Word_Embeddings_in_Title.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create vector representations of titles in dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import nltk, collections, re, string\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import SnowballStemmer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('Eluvio_DS_Challenge.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time_created</th>\n",
       "      <th>date_created</th>\n",
       "      <th>up_votes</th>\n",
       "      <th>down_votes</th>\n",
       "      <th>title</th>\n",
       "      <th>over_18</th>\n",
       "      <th>author</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1201232046</td>\n",
       "      <td>2008-01-25</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>Scores killed in Pakistan clashes</td>\n",
       "      <td>False</td>\n",
       "      <td>polar</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1201232075</td>\n",
       "      <td>2008-01-25</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>Japan resumes refuelling mission</td>\n",
       "      <td>False</td>\n",
       "      <td>polar</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1201232523</td>\n",
       "      <td>2008-01-25</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>US presses Egypt on Gaza border</td>\n",
       "      <td>False</td>\n",
       "      <td>polar</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1201233290</td>\n",
       "      <td>2008-01-25</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Jump-start economy: Give health care to all</td>\n",
       "      <td>False</td>\n",
       "      <td>fadi420</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1201274720</td>\n",
       "      <td>2008-01-25</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>Council of Europe bashes EU&amp;UN terror blacklist</td>\n",
       "      <td>False</td>\n",
       "      <td>mhermans</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509231</th>\n",
       "      <td>1479816764</td>\n",
       "      <td>2016-11-22</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>Heil Trump : Donald Trump s  alt-right  white...</td>\n",
       "      <td>False</td>\n",
       "      <td>nonamenoglory</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509232</th>\n",
       "      <td>1479816772</td>\n",
       "      <td>2016-11-22</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>There are people speculating that this could b...</td>\n",
       "      <td>False</td>\n",
       "      <td>SummerRay</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509233</th>\n",
       "      <td>1479817056</td>\n",
       "      <td>2016-11-22</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Professor receives Arab Researchers Award</td>\n",
       "      <td>False</td>\n",
       "      <td>AUSharjah</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509234</th>\n",
       "      <td>1479817157</td>\n",
       "      <td>2016-11-22</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Nigel Farage attacks response to Trump ambassa...</td>\n",
       "      <td>False</td>\n",
       "      <td>smilyflower</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509235</th>\n",
       "      <td>1479817346</td>\n",
       "      <td>2016-11-22</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Palestinian wielding knife shot dead in West B...</td>\n",
       "      <td>False</td>\n",
       "      <td>superislam</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>509236 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        time_created date_created  up_votes  down_votes  \\\n",
       "0         1201232046   2008-01-25         3           0   \n",
       "1         1201232075   2008-01-25         2           0   \n",
       "2         1201232523   2008-01-25         3           0   \n",
       "3         1201233290   2008-01-25         1           0   \n",
       "4         1201274720   2008-01-25         4           0   \n",
       "...              ...          ...       ...         ...   \n",
       "509231    1479816764   2016-11-22         5           0   \n",
       "509232    1479816772   2016-11-22         1           0   \n",
       "509233    1479817056   2016-11-22         1           0   \n",
       "509234    1479817157   2016-11-22         1           0   \n",
       "509235    1479817346   2016-11-22         1           0   \n",
       "\n",
       "                                                    title  over_18  \\\n",
       "0                       Scores killed in Pakistan clashes    False   \n",
       "1                        Japan resumes refuelling mission    False   \n",
       "2                         US presses Egypt on Gaza border    False   \n",
       "3            Jump-start economy: Give health care to all     False   \n",
       "4         Council of Europe bashes EU&UN terror blacklist    False   \n",
       "...                                                   ...      ...   \n",
       "509231   Heil Trump : Donald Trump s  alt-right  white...    False   \n",
       "509232  There are people speculating that this could b...    False   \n",
       "509233          Professor receives Arab Researchers Award    False   \n",
       "509234  Nigel Farage attacks response to Trump ambassa...    False   \n",
       "509235  Palestinian wielding knife shot dead in West B...    False   \n",
       "\n",
       "               author   category  \n",
       "0               polar  worldnews  \n",
       "1               polar  worldnews  \n",
       "2               polar  worldnews  \n",
       "3             fadi420  worldnews  \n",
       "4            mhermans  worldnews  \n",
       "...               ...        ...  \n",
       "509231  nonamenoglory  worldnews  \n",
       "509232      SummerRay  worldnews  \n",
       "509233      AUSharjah  worldnews  \n",
       "509234    smilyflower  worldnews  \n",
       "509235     superislam  worldnews  \n",
       "\n",
       "[509236 rows x 8 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### clean titles "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "punctuation_numbers_to_exclude = r\"[\\d+{}’‘“”…£]\".format(string.punctuation)\n",
    "stop_words = [re.sub(punctuation_numbers_to_exclude, \"\", stop) for stop in stopwords.words('english')]\n",
    "ss = SnowballStemmer('english')\n",
    "\n",
    "def clean_text(text, remove_punctuation=True, remove_stopwords=True, stem=True):\n",
    "    \n",
    "    text=text.lower()\n",
    "    \n",
    "    if remove_punctuation:\n",
    "        #remove punctuation and numbers\n",
    "        text = re.sub(punctuation_numbers_to_exclude, \"\", text)\n",
    "    \n",
    "    if remove_stopwords:\n",
    "        #remove stopwords\n",
    "        text = [word for word in text.split() if word not in stop_words]\n",
    "   \n",
    "    if stem:\n",
    "        #stem\n",
    "        text = [ss.stem(word) for word in text]\n",
    "    \n",
    "    return ' '.join(text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_titles_no_stemming = [clean_text(title, stem=False) for title in data['title'].values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Scores killed in Pakistan clashes',\n",
       "       'Japan resumes refuelling mission',\n",
       "       'US presses Egypt on Gaza border',\n",
       "       'Jump-start economy: Give health care to all ',\n",
       "       'Council of Europe bashes EU&UN terror blacklist',\n",
       "       'Hay presto! Farmer unveils the  illegal  mock-Tudor castle he tried to hide behind 40ft hay bales',\n",
       "       'Strikes, Protests and Gridlock at the Poland-Ukraine Border',\n",
       "       'The U.N. Mismanagement Program',\n",
       "       'Nicolas Sarkozy threatens to sue Ryanair ',\n",
       "       'US plans for missile shields in Polish town met with resistance [video]'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['title'].values[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['scores killed pakistan clashes',\n",
       " 'japan resumes refuelling mission',\n",
       " 'us presses egypt gaza border',\n",
       " 'jumpstart economy give health care',\n",
       " 'council europe bashes euun terror blacklist',\n",
       " 'hay presto farmer unveils illegal mocktudor castle tried hide behind ft hay bales',\n",
       " 'strikes protests gridlock polandukraine border',\n",
       " 'un mismanagement program',\n",
       " 'nicolas sarkozy threatens sue ryanair',\n",
       " 'us plans missile shields polish town met resistance video']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_titles_no_stemming[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use pretrained [GloVe](https://nlp.stanford.edu/projects/glove/) model to create vector embeddings for words in titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load pretrained model\n",
    "\n",
    "embeddings_dict = {}\n",
    "dimensions = 50\n",
    "with open(\"glove.6b/glove.6B.50d.txt\", 'r') as f:\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        vector = np.asarray(values[1:], \"float32\")\n",
    "        embeddings_dict[word] = vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create array of vector embeddings from titles."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles_vectors = []\n",
    "\n",
    "#using nonstemming because word variants with the same stem often appear with separate embeddings.\n",
    "for title in cleaned_titles_no_stemming:\n",
    "    title_words_vectors = []\n",
    "    for word in title.lower().split():\n",
    "        try:\n",
    "            title_words_vectors.append(embeddings_dict[word])\n",
    "        except KeyError: #if word not in model, then pass\n",
    "            pass\n",
    "    \n",
    "    #if there are no words from title in GloVe model, return array of nans, else return mean of embedding values for all words in title\n",
    "    if len(title_words_vectors) == 0:\n",
    "        titles_vectors.append(np.full(dimensions, np.nan))\n",
    "    else:\n",
    "        titles_vectors.append(np.mean(title_words_vectors, axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 3.9545023e-01, -4.8771000e-01,  5.7768625e-01, -9.1156498e-02,\n",
       "        1.8073300e-01,  1.7479993e-02, -2.5747500e-02,  5.7215750e-01,\n",
       "        3.0598015e-02, -4.1651750e-01, -3.6815000e-01, -1.3932850e+00,\n",
       "       -6.4465249e-01, -4.2249000e-01,  2.8884250e-01, -1.5836248e-01,\n",
       "        8.9348249e-02, -1.4332525e-01, -8.2170749e-01,  5.5123746e-01,\n",
       "       -2.1498752e-01,  9.0665251e-01,  6.1633497e-01,  3.8069525e-01,\n",
       "       -6.0425989e-02, -1.3075874e+00, -6.3089997e-02, -4.4174001e-01,\n",
       "       -9.9862486e-02,  3.5555500e-01,  2.6287999e+00,  2.5334752e-01,\n",
       "       -3.7351996e-02, -1.0227509e-02,  6.0472500e-01,  6.7483252e-01,\n",
       "        9.7777516e-02, -9.7192156e-01, -5.1433748e-01,  5.6634498e-01,\n",
       "       -6.8620503e-01,  5.3959477e-01,  1.1125749e-01, -3.3677000e-01,\n",
       "        8.0702752e-01,  1.8408252e-01, -5.1430500e-01,  4.4885755e-01,\n",
       "        1.1374950e-03, -1.1982850e+00], dtype=float32)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titles_vectors[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## simple check by finding most similar title using smallest distance in embedding space "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.spatial.distance import pdist, squareform\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sample = titles_vectors[:10000]\n",
    "\n",
    "#mask out titles that had no words in GloVe and are represented by nans\n",
    "nan_mask = np.any(np.isfinite(test_sample), axis=1)\n",
    "\n",
    "distances = pdist(np.array(test_sample)[nan_mask])\n",
    "\n",
    "#convert to square matrix\n",
    "distances_square = squareform(distances)\n",
    "\n",
    "#diagonals represent a distance of 0, so replace with infinity so it is not selected as shortest distance\n",
    "np.fill_diagonal(distances_square, np.inf)\n",
    "\n",
    "closest_points = distances_square.argmin(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([8489,  763,   55, ..., 8227, 1131, 4048])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "closest_points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "def most_similar_title(index, sample_length=10000):\n",
    "    titles = data['title'][:sample_length].values\n",
    "    \n",
    "    print('- Title: {0} \\n'.format(titles[nan_mask][index]))\n",
    "    \n",
    "    closest_title = titles[nan_mask][closest_points[index]]\n",
    "    \n",
    "    print('- Closest Title: {0}\\n'.format(closest_title))\n",
    "    print('--------------------------------\\n')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Title: Cruising for chicks, Saudi Arabian style \n",
      "\n",
      "- Closest Title: Hamburger Hill: frontline base in a sea of poppies British soldiers dare not clear\n",
      "\n",
      "--------------------------------\n",
      "\n",
      "- Title: Tibetan protesters beaten up in Nepal, 125 detained \n",
      "\n",
      "- Closest Title: Police Beat Tibetan Protesters in Nepal\n",
      "\n",
      "--------------------------------\n",
      "\n",
      "- Title: The Price of Faithlessness: Iran to Punish Apostasy with Death \n",
      "\n",
      "- Closest Title: Iranian woman escapes stoning to death for adultery - did social media sway the Iranians?\n",
      "\n",
      "--------------------------------\n",
      "\n",
      "- Title: Amazing picture of the moment a British teacher was attacked by a lion \n",
      "\n",
      "- Closest Title: Google rats on Indian expressionist: Big Brother is watching all of us\n",
      "\n",
      "--------------------------------\n",
      "\n",
      "- Title:  If he (Mr Bush) can say he has killed Saddam Hussein and captured bin Laden, he can claim to have left the world a safer place  \n",
      "\n",
      "- Closest Title: If Bin Laden is Still Alive and Still a Threat, and if Bush is All About Winning the  War on Terror , Why is Bush Only NOW Showing Re-Found Interest in Killing Him?\n",
      "\n",
      "--------------------------------\n",
      "\n",
      "- Title: EU has no clue who runs its foreign policy \n",
      "\n",
      "- Closest Title: Turkey s intense diplomacy overturned France s efforts and the word  accession  was re-added to the document, approved by the EU foreign ministers \n",
      "\n",
      "--------------------------------\n",
      "\n",
      "- Title: China, Russia, U.S. focus of human rights report \n",
      "\n",
      "- Closest Title: Japan: Press China’s President to Protect Human Rights (Human Rights Watch)\n",
      "\n",
      "--------------------------------\n",
      "\n",
      "- Title: 6 Reasons to Visit the World s Happiest Country \n",
      "\n",
      "- Closest Title: Why Israel is the world s happiest country\n",
      "\n",
      "--------------------------------\n",
      "\n",
      "- Title: Pakistan rally, unexpectedly huge, presses judges  cause as mullahs and communists, and everything in between, came out in Pakistan Friday in a massive rally against Musharraf \n",
      "\n",
      "- Closest Title: Nepal s Maoists said Monday they were pouring tens of thousands of members of their feared youth wing into the capital ahead of this week s anticipated abolition of the monarchy.\n",
      "\n",
      "--------------------------------\n",
      "\n",
      "- Title: Earthquake Death Toll in China May Be 5,000 \n",
      "\n",
      "- Closest Title:  China says death toll in quake more than 51,000\n",
      "\n",
      "--------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in random.sample(range(10000), 10):\n",
    "    most_similar_title(i)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Does pretty well for some titles, not great for others. Not sure how useful it will prove to be for predicting upvotes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:eluvio]",
   "language": "python",
   "name": "conda-env-eluvio-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}