Week_10/Day_1/Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import copy\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>adult</th>\n",
       "      <th>belongs_to_collection</th>\n",
       "      <th>budget</th>\n",
       "      <th>genres</th>\n",
       "      <th>homepage</th>\n",
       "      <th>id</th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>original_language</th>\n",
       "      <th>original_title</th>\n",
       "      <th>overview</th>\n",
       "      <th>...</th>\n",
       "      <th>release_date</th>\n",
       "      <th>revenue</th>\n",
       "      <th>runtime</th>\n",
       "      <th>spoken_languages</th>\n",
       "      <th>status</th>\n",
       "      <th>tagline</th>\n",
       "      <th>title</th>\n",
       "      <th>video</th>\n",
       "      <th>vote_average</th>\n",
       "      <th>vote_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 10194, 'name': 'Toy Story Collection', ...</td>\n",
       "      <td>30000000</td>\n",
       "      <td>[{'id': 16, 'name': 'Animation'}, {'id': 35, '...</td>\n",
       "      <td>http://toystory.disney.com/toy-story</td>\n",
       "      <td>862</td>\n",
       "      <td>tt0114709</td>\n",
       "      <td>en</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>Led by Woody, Andy's toys live happily in his ...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-10-30</td>\n",
       "      <td>373554033.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>False</td>\n",
       "      <td>7.7</td>\n",
       "      <td>5415.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>65000000</td>\n",
       "      <td>[{'id': 12, 'name': 'Adventure'}, {'id': 14, '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8844</td>\n",
       "      <td>tt0113497</td>\n",
       "      <td>en</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>When siblings Judy and Peter discover an encha...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-12-15</td>\n",
       "      <td>262797249.0</td>\n",
       "      <td>104.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}, {'iso...</td>\n",
       "      <td>Released</td>\n",
       "      <td>Roll the dice and unleash the excitement!</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>False</td>\n",
       "      <td>6.9</td>\n",
       "      <td>2413.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 119050, 'name': 'Grumpy Old Men Collect...</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'id': 10749, 'name': 'Romance'}, {'id': 35, ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15602</td>\n",
       "      <td>tt0113228</td>\n",
       "      <td>en</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>A family wedding reignites the ancient feud be...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-12-22</td>\n",
       "      <td>0.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>Still Yelling. Still Fighting. Still Ready for...</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>False</td>\n",
       "      <td>6.5</td>\n",
       "      <td>92.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   adult                              belongs_to_collection    budget  \\\n",
       "0  False  {'id': 10194, 'name': 'Toy Story Collection', ...  30000000   \n",
       "1  False                                                NaN  65000000   \n",
       "2  False  {'id': 119050, 'name': 'Grumpy Old Men Collect...         0   \n",
       "\n",
       "                                              genres  \\\n",
       "0  [{'id': 16, 'name': 'Animation'}, {'id': 35, '...   \n",
       "1  [{'id': 12, 'name': 'Adventure'}, {'id': 14, '...   \n",
       "2  [{'id': 10749, 'name': 'Romance'}, {'id': 35, ...   \n",
       "\n",
       "                               homepage     id    imdb_id original_language  \\\n",
       "0  http://toystory.disney.com/toy-story    862  tt0114709                en   \n",
       "1                                   NaN   8844  tt0113497                en   \n",
       "2                                   NaN  15602  tt0113228                en   \n",
       "\n",
       "     original_title                                           overview  ...  \\\n",
       "0         Toy Story  Led by Woody, Andy's toys live happily in his ...  ...   \n",
       "1           Jumanji  When siblings Judy and Peter discover an encha...  ...   \n",
       "2  Grumpier Old Men  A family wedding reignites the ancient feud be...  ...   \n",
       "\n",
       "  release_date      revenue runtime  \\\n",
       "0   1995-10-30  373554033.0    81.0   \n",
       "1   1995-12-15  262797249.0   104.0   \n",
       "2   1995-12-22          0.0   101.0   \n",
       "\n",
       "                                    spoken_languages    status  \\\n",
       "0           [{'iso_639_1': 'en', 'name': 'English'}]  Released   \n",
       "1  [{'iso_639_1': 'en', 'name': 'English'}, {'iso...  Released   \n",
       "2           [{'iso_639_1': 'en', 'name': 'English'}]  Released   \n",
       "\n",
       "                                             tagline             title  video  \\\n",
       "0                                                NaN         Toy Story  False   \n",
       "1          Roll the dice and unleash the excitement!           Jumanji  False   \n",
       "2  Still Yelling. Still Fighting. Still Ready for...  Grumpier Old Men  False   \n",
       "\n",
       "  vote_average vote_count  \n",
       "0          7.7     5415.0  \n",
       "1          6.9     2413.0  \n",
       "2          6.5       92.0  \n",
       "\n",
       "[3 rows x 24 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load Movies Metadata\n",
    "metadata = pd.read_csv('movies_metadata.csv', low_memory=False)\n",
    "\n",
    "# Print the first three rows\n",
    "metadata.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"weighted_rating.png\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.618207215134185\n"
     ]
    }
   ],
   "source": [
    "C = metadata['vote_average'].mean()\n",
    "print(C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "160.0\n"
     ]
    }
   ],
   "source": [
    "m = metadata['vote_count'].quantile(0.90)\n",
    "print(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4555, 24)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_movies = metadata.copy().loc[metadata['vote_count'] >= m]\n",
    "q_movies.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(45466, 24)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weighted_rating(x, m=m, C=C):\n",
    "    v = x['vote_count']\n",
    "    R = x['vote_average']\n",
    "    # Calculation based on the IMDB formula\n",
    "    return (v/(v+m) * R) + (m/(m+v) * C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "q_movies['score'] = q_movies.apply(weighted_rating, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>vote_count</th>\n",
       "      <th>vote_average</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>314</th>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>8358.0</td>\n",
       "      <td>8.5</td>\n",
       "      <td>8.445869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>834</th>\n",
       "      <td>The Godfather</td>\n",
       "      <td>6024.0</td>\n",
       "      <td>8.5</td>\n",
       "      <td>8.425439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10309</th>\n",
       "      <td>Dilwale Dulhania Le Jayenge</td>\n",
       "      <td>661.0</td>\n",
       "      <td>9.1</td>\n",
       "      <td>8.421453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12481</th>\n",
       "      <td>The Dark Knight</td>\n",
       "      <td>12269.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.265477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2843</th>\n",
       "      <td>Fight Club</td>\n",
       "      <td>9678.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.256385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>292</th>\n",
       "      <td>Pulp Fiction</td>\n",
       "      <td>8670.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.251406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>522</th>\n",
       "      <td>Schindler's List</td>\n",
       "      <td>4436.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.206639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23673</th>\n",
       "      <td>Whiplash</td>\n",
       "      <td>4376.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.205404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5481</th>\n",
       "      <td>Spirited Away</td>\n",
       "      <td>3968.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.196055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2211</th>\n",
       "      <td>Life Is Beautiful</td>\n",
       "      <td>3643.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.187171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1178</th>\n",
       "      <td>The Godfather: Part II</td>\n",
       "      <td>3418.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.180076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1152</th>\n",
       "      <td>One Flew Over the Cuckoo's Nest</td>\n",
       "      <td>3001.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.164256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351</th>\n",
       "      <td>Forrest Gump</td>\n",
       "      <td>8147.0</td>\n",
       "      <td>8.2</td>\n",
       "      <td>8.150272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1154</th>\n",
       "      <td>The Empire Strikes Back</td>\n",
       "      <td>5998.0</td>\n",
       "      <td>8.2</td>\n",
       "      <td>8.132919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1176</th>\n",
       "      <td>Psycho</td>\n",
       "      <td>2405.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.132715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18465</th>\n",
       "      <td>The Intouchables</td>\n",
       "      <td>5410.0</td>\n",
       "      <td>8.2</td>\n",
       "      <td>8.125837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40251</th>\n",
       "      <td>Your Name.</td>\n",
       "      <td>1030.0</td>\n",
       "      <td>8.5</td>\n",
       "      <td>8.112532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>Leon: The Professional</td>\n",
       "      <td>4293.0</td>\n",
       "      <td>8.2</td>\n",
       "      <td>8.107234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3030</th>\n",
       "      <td>The Green Mile</td>\n",
       "      <td>4166.0</td>\n",
       "      <td>8.2</td>\n",
       "      <td>8.104511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1170</th>\n",
       "      <td>GoodFellas</td>\n",
       "      <td>3211.0</td>\n",
       "      <td>8.2</td>\n",
       "      <td>8.077459</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 title  vote_count  vote_average     score\n",
       "314           The Shawshank Redemption      8358.0           8.5  8.445869\n",
       "834                      The Godfather      6024.0           8.5  8.425439\n",
       "10309      Dilwale Dulhania Le Jayenge       661.0           9.1  8.421453\n",
       "12481                  The Dark Knight     12269.0           8.3  8.265477\n",
       "2843                        Fight Club      9678.0           8.3  8.256385\n",
       "292                       Pulp Fiction      8670.0           8.3  8.251406\n",
       "522                   Schindler's List      4436.0           8.3  8.206639\n",
       "23673                         Whiplash      4376.0           8.3  8.205404\n",
       "5481                     Spirited Away      3968.0           8.3  8.196055\n",
       "2211                 Life Is Beautiful      3643.0           8.3  8.187171\n",
       "1178            The Godfather: Part II      3418.0           8.3  8.180076\n",
       "1152   One Flew Over the Cuckoo's Nest      3001.0           8.3  8.164256\n",
       "351                       Forrest Gump      8147.0           8.2  8.150272\n",
       "1154           The Empire Strikes Back      5998.0           8.2  8.132919\n",
       "1176                            Psycho      2405.0           8.3  8.132715\n",
       "18465                 The Intouchables      5410.0           8.2  8.125837\n",
       "40251                       Your Name.      1030.0           8.5  8.112532\n",
       "289             Leon: The Professional      4293.0           8.2  8.107234\n",
       "3030                    The Green Mile      4166.0           8.2  8.104511\n",
       "1170                        GoodFellas      3211.0           8.2  8.077459"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_movies = q_movies.sort_values('score', ascending=False)\n",
    "\n",
    "#Print the top 15 movies\n",
    "q_movies[['title', 'vote_count', 'vote_average', 'score']].head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Content based recommender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(45466, 75827)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Define a TF-IDF Vectorizer Object. Remove all english stop words such as 'the', 'a'\n",
    "tfidf = TfidfVectorizer(stop_words='english')\n",
    "\n",
    "#Replace NaN with an empty string\n",
    "metadata['overview'] = metadata['overview'].fillna('')\n",
    "\n",
    "#Construct the required TF-IDF matrix by fitting and transforming the data\n",
    "tfidf_matrix = tfidf.fit_transform(metadata['overview'])\n",
    "\n",
    "#Output the shape of tfidf_matrix\n",
    "tfidf_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['avails',\n",
       " 'avaks',\n",
       " 'avalanche',\n",
       " 'avalanches',\n",
       " 'avallone',\n",
       " 'avalon',\n",
       " 'avant',\n",
       " 'avanthika',\n",
       " 'avanti',\n",
       " 'avaracious']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf.get_feature_names()[5000:5010]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}