Model Testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sqlite3\n",
    "import gensim\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n",
    "import matplotlib\n",
    "from gensim.models import word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sql_conn = sqlite3.connect('../data/database.sqlite')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# These functions are needed for processing later\n",
    "\n",
    "# Takes a sentence in a comment and converts it to a list of words.\n",
    "def comment_to_wordlist(comment, remove_stopwords=False ):\n",
    "    comment = re.sub(\"[^a-zA-Z]\",\" \", comment)\n",
    "    words = comment.lower().split()\n",
    "    if remove_stopwords:\n",
    "        stops = set(stopwords.words(\"english\"))\n",
    "        words = [w for w in words if not w in stops]\n",
    "\n",
    "    return(words)\n",
    "\n",
    "# Takes a comment and converts it to an array of sentences\n",
    "def comment_to_sentence(comment, tokenizer, remove_stopwords=False):\n",
    "    raw_sentences = tokenizer.tokenize(comment.strip())\n",
    "    \n",
    "    sentences = []\n",
    "    for s in raw_sentences:\n",
    "        if len(s)>0:\n",
    "            sentences.append(comment_to_wordlist(s, remove_stopwords))\n",
    "    #rof\n",
    "    return sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Choice of subreddits\n",
    "\n",
    "These subreddits contain discussions on 12 fairly distinct topic, from a human point of view."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mathematics = pd.read_sql(\"SELECT subreddit, body FROM May2015 WHERE subreddit == 'mathematics'\",sql_conn)\n",
    "\n",
    "computerscience = pd.read_sql(\"SELECT subreddit, body FROM May2015 WHERE subreddit == 'computerscience'\",sql_conn)\n",
    "\n",
    "history = pd.read_sql(\"SELECT subreddit, body FROM May2015 WHERE subreddit == 'history'\",sql_conn)\n",
    "\n",
    "philosophy = pd.read_sql(\"SELECT subreddit, body FROM May2015 WHERE subreddit == 'philosophy'\",sql_conn)\n",
    "\n",
    "elifive = pd.read_sql(\"SELECT subreddit, body FROM May2015 WHERE subreddit == 'explainlikeimfive'\",sql_conn)\n",
    "\n",
    "askanthro = pd.read_sql(\"SELECT subreddit, body FROM May2015 WHERE subreddit == 'AskAnthropology'\",sql_conn)\n",
    "\n",
    "homebrewing = pd.read_sql(\"SELECT subreddit, body FROM May2015 WHERE subreddit == 'Homebrewing'\",sql_conn)\n",
    "\n",
    "bicycling = pd.read_sql(\"SELECT subreddit, body FROM May2015 WHERE subreddit == 'bicycling'\", sql_conn)\n",
    "\n",
    "food = pd.read_sql(\"SELECT subreddit, body FROM May2015 WHERE subreddit == 'food'\", sql_conn)\n",
    "\n",
    "science = pd.read_sql(\"SELECT subreddit, body FROM May2015 WHERE subreddit == 'science'\", sql_conn)\n",
    "\n",
    "movies = pd.read_sql(\"SELECT subreddit, body FROM May2015 WHERE subreddit == 'movies'\", sql_conn)\n",
    "\n",
    "books = pd.read_sql(\"SELECT subreddit, body FROM May2015 WHERE subreddit == 'books'\", sql_conn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Array of tuples, with df and subject\n",
    "subreddits = [(bicycling,'bicycling'),(history,'history'),(philosophy,'philosophy'),\n",
    "              (elifive,'explain'),(homebrewing,'homebrew'),(askanthro,'anthropology'),\n",
    "              (mathematics,'mathematics'),(computerscience,'computer'),\n",
    "              (food,\"food\"),(science,\"science\"),(movies,\"movies\"),(books,\"books\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 931315 entries, 0 to 931314\n",
      "Data columns (total 2 columns):\n",
      "subreddit    931315 non-null object\n",
      "body         931315 non-null object\n",
      "dtypes: object(2)\n",
      "memory usage: 21.3+ MB\n"
     ]
    }
   ],
   "source": [
    "all_frames = [bicycling, history, philosophy, elifive, homebrewing, askanthro, mathematics,\\\n",
    "              computerscience, food, science, movies, books]\n",
    "model_training_data = pd.concat(all_frames, ignore_index=True)\n",
    "model_training_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['300features_50minwords_10context', '300features_100minwords_10context', '300features_10minwords_3context', '300features_100minwords_5context', '300features_10minwords_5context', '300features_50minwords_5context', '300features_30minwords_5context', '300features_100minwords_3context', '300features_10minwords_10context', '300features_30minwords_3context', '300features_50minwords_3context', '300features_30minwords_10context']\n"
     ]
    }
   ],
   "source": [
    "# Get all model files\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "models = [f for f in listdir('new_models/') if isfile(join('new_models/', f)) and not f.endswith('.npy')]\n",
    "print(models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "# Method for computing similarity to a specific label\n",
    "# Note this returns the cosine similarity between \n",
    "# the vector representation of the words and label\n",
    "def similarities(comment, label, model):\n",
    "    dists = []\n",
    "    for word in comment:\n",
    "        if word in model.vocab:\n",
    "            dists.append(model.similarity(word,label))\n",
    "    return dists\n",
    "\n",
    "def label_comment(comment, labels, model):\n",
    "    # Set initial distance to be 1-(-1) (complete dissimilarity)\n",
    "    best_distance = 2\n",
    "    best_label = \"\"\n",
    "    for label in labels:\n",
    "        # Range will be from [-1,1]\n",
    "        word_dists_to_label = similarities(comment, label, model)\n",
    "        \n",
    "        # We want to choose the label with overall sum closest to 1\n",
    "        # or\n",
    "        # We want to minimize our distance to 1\n",
    "        dist = 1 - sum(word_dists_to_label)\n",
    "        if dist < best_distance:\n",
    "            best_label = label\n",
    "            best_distance = dist\n",
    "    return best_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300features_50minwords_10context\n",
      "Size of vocab: 19859\n",
      "300features_100minwords_10context\n",
      "Size of vocab: 13593\n",
      "300features_10minwords_3context\n",
      "Size of vocab: 43723\n",
      "300features_100minwords_5context\n",
      "Size of vocab: 13593\n",
      "300features_10minwords_5context\n",
      "Size of vocab: 43723\n",
      "300features_50minwords_5context\n",
      "Size of vocab: 19859\n",
      "300features_30minwords_5context\n",
      "Size of vocab: 25879\n",
      "300features_100minwords_3context\n",
      "Size of vocab: 13593\n",
      "300features_10minwords_10context\n",
      "Size of vocab: 43723\n",
      "300features_30minwords_3context\n",
      "Size of vocab: 25879\n",
      "300features_50minwords_3context\n",
      "Size of vocab: 19859\n",
      "300features_30minwords_10context\n",
      "Size of vocab: 25879\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import word2vec\n",
    "\n",
    "# Vocab size\n",
    "\n",
    "# Attempt to load each model\n",
    "# Display size of vocabulary for each model\n",
    "# As expected, lowering the minimum number of appearances of a word increases\n",
    "# the size of the vocabulary\n",
    "\n",
    "xs = []\n",
    "size_of_vocab=[]\n",
    "\n",
    "for m in models:\n",
    "    print(m)\n",
    "    current_model = word2vec.Word2Vec.load('new_models/' + m);\n",
    "    print(\"Size of vocab: \" + str(current_model.syn0.shape[0]))\n",
    "    \n",
    "    x = m.split(\"_\")[1]\n",
    "    xs.append(x.split(\"min\")[0])\n",
    "    size_of_vocab.append(current_model.syn0.shape[0])\n",
    "    \n",
    "    # Take each comment, compare "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "m = \"300features_10minwords_10context\"\n",
    "current_model = word2vec.Word2Vec.load('new_models/' + m);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'proofs', 0.7183476090431213),\n",
       " (u'mathematical', 0.6981117725372314),\n",
       " (u'math', 0.6802877187728882),\n",
       " (u'maths', 0.6618346571922302),\n",
       " (u'godel', 0.6530870199203491),\n",
       " (u'calculus', 0.6464293003082275),\n",
       " (u'automata', 0.637760579586029),\n",
       " (u'physics', 0.6259437203407288),\n",
       " (u'axioms', 0.6245019435882568),\n",
       " (u'equations', 0.6236181259155273)]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current_model.most_similar(\"mathematics\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Including r/movies into my data created some interesting associations. People on the internet are mean."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "\n",
    "comments = []\n",
    "ground_truth_labels = []\n",
    "for i in range(len(model_training_data)):\n",
    "    comments.append(comment_to_wordlist(model_training_data.iloc[i]['body'], tokenizer))\n",
    "    ground_truth_labels.append(model_training_data.iloc[i]['subreddit']) # Could also just slice this array out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Length is the same: ', 999300)\n"
     ]
    }
   ],
   "source": [
    "if len(comments)==len(ground_truth_labels):\n",
    "    print(\"Length is the same: \",len(comments))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#labels = np.unique(model_training_data['subreddit'])\n",
    "#subreddits = [(bicycling,'bicycling'),(history,'history'),(philosophy,'philosophy'),\n",
    "#              (elifive,'explain'),(homebrewing,'homebrew'),(askanthro,'anthropology'),\n",
    "#              (mathematics,'mathematics'),(computerscience,'computer science'),\n",
    "#              (food,\"food\"),(gaming,\"gaming\"),(politics,\"politics\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "subreddit_labels = []\n",
    "for frame,name in subreddits:\n",
    "    subreddit_labels.append(name)\n",
    "    \n",
    "trained_labels = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "#trained_labels = [None] * len(comments)\n",
    "start = time.time()\n",
    "for i in range(100000):\n",
    "    trained_labels.append(label_comment(comments[i],subreddit_labels,current_model))\n",
    "end = time.time()\n",
    "duration = end-start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Took 617.820972919 time to label 100000 comments.\n"
     ]
    }
   ],
   "source": [
    "print(\"Took \" + str(duration) + \" time to label 100000 comments.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200000"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trained_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mis_rate = 0\n",
    "successes = []\n",
    "for i in range(len(trained_labels)):\n",
    "    if trained_labels[i] != ground_truth_labels[i]:\n",
    "        mis_rate += 1\n",
    "    else:\n",
    "        successes.append((comments[i],trained_labels[i]))\n",
    "error_rate = mis_rate/float(len(trained_labels))\n",
    "success_rate = 1-error_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24341"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(successes)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}