get_imdb_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "colab": {
      "name": "get_imdb_data.ipynb",
      "provenance": []
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jSKlG-WNHEjF",
        "outputId": "e66a7d28-47d8-47d6-ede1-b331e4c7e058"
      },
      "source": [
        "!pip install -U beautifulsoup4"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already up-to-date: beautifulsoup4 in /usr/local/lib/python3.7/dist-packages (4.9.3)\n",
            "Requirement already satisfied, skipping upgrade: soupsieve>1.2; python_version >= \"3.0\" in /usr/local/lib/python3.7/dist-packages (from beautifulsoup4) (2.2.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jadTEy3W4zls"
      },
      "source": [
        "import sqlite3\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "from tqdm import tqdm\n",
        "from datetime import datetime"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4azjY4tH4zlu"
      },
      "source": [
        "class IMDBScraper:\n",
        "    def get_top_rated(self):\n",
        "        request_headers = {\n",
        "            'accept-language': \"en-US,en;q=1.0\",\n",
        "            'content-language': 'en-US',\n",
        "            'user-agent': 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36'\n",
        "        }\n",
        "        response = requests.get('https://www.imdb.com/chart/top', headers=request_headers)\n",
        "        soup = BeautifulSoup(response.text)\n",
        "        movie_titles = [e.text for e in soup.select('.titleColumn a')]\n",
        "        movie_years = [int(e.text.replace('(', '').replace(')', '')) for e in soup.select('.secondaryInfo')]\n",
        "        top_rated_movies = pd.DataFrame()\n",
        "        top_rated_movies['id'] = range(1, len(movie_titles) + 1)\n",
        "        top_rated_movies['title'] = movie_titles\n",
        "        top_rated_movies['release_year'] = movie_years\n",
        "        movie_page_links = [\"https://www.imdb.com{}\".format(e.get(\"href\")) for e in soup.select('.titleColumn a')]\n",
        "        self.movie_page_links = movie_page_links\n",
        "        return top_rated_movies\n",
        "    def get_movie_info(self):\n",
        "        top_rated_movies = self.get_top_rated()\n",
        "        movie_ids = top_rated_movies['id'].values\n",
        "        movie_info = []\n",
        "        for link, mid in tqdm(zip(self.movie_page_links, movie_ids)):\n",
        "            request_headers = {\n",
        "            'accept-language': \"en-US,en;q=1.0\",\n",
        "            'content-language': 'en-US',\n",
        "            'user-agent': 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36'\n",
        "            }\n",
        "            response = requests.get(link, headers=request_headers)\n",
        "            soup = BeautifulSoup(response.text)\n",
        "            rating = float(soup.select('strong span')[0].text)\n",
        "            director = soup.select('.credit_summary_item a')[0].text\n",
        "            genre = [e.text.strip() for e in soup.select('.subtext a')]\n",
        "            release_date = genre.pop()\n",
        "            release_date_split = release_date.split()\n",
        "            release_date_split.pop()\n",
        "            release_date = '-'.join(release_date_split)\n",
        "            movie_time = soup.select('time')[0].text.strip().split()\n",
        "            if len(movie_time) == 2:\n",
        "                hours, mins = movie_time\n",
        "                hours = int(hours.replace('h', ''))\n",
        "                mins = int(mins.replace('min', ''))\n",
        "                movie_time_mins = hours * 60 + mins\n",
        "            else:\n",
        "                hours_mins = movie_time[0]\n",
        "                if 'h' in hours_mins:\n",
        "                    hours = int(hours_mins.replace('h', ''))\n",
        "                    movie_time_mins = hours * 60\n",
        "                elif 'min' in hours_mins:\n",
        "                    mins = int(hours_mins.replace('min', ''))\n",
        "                    movie_time_mins = mins\n",
        "            actor = [e.text.strip() for e in soup.select('.primary_photo+ td a')]\n",
        "            ord = list(range(1, len(actor) + 1))\n",
        "            movie_data = {\n",
        "                'id': mid,\n",
        "                'rating': rating,\n",
        "                'director': director,\n",
        "                'genre': genre,\n",
        "                'release_date': release_date,\n",
        "                'runtime': movie_time_mins,\n",
        "                'actor': actor,\n",
        "                'ord': ord\n",
        "            }\n",
        "            movie_info.append(movie_data)\n",
        "        self._movie_info = movie_info\n",
        "        return movie_info\n",
        "    def get_top_rated_movies(self):\n",
        "        top_rated = self.get_top_rated()\n",
        "        movie_info = self.get_movie_info()\n",
        "        movie_info_list = []\n",
        "        for item in movie_info:\n",
        "            movie_data = {\n",
        "                'id': item['id'],\n",
        "                'rating': item['rating'],\n",
        "                'director': item['director'],\n",
        "                'runtime': item['runtime']\n",
        "            }\n",
        "            movie_info_list.append(movie_data)\n",
        "        movie_info_df = pd.DataFrame(movie_info_list)\n",
        "        top_rated_movies = pd.merge(top_rated, movie_info_df, left_on='id', right_on='id')\n",
        "        return top_rated_movies[['id', 'title', 'release_year', 'rating', 'director', 'runtime']]\n",
        "    def get_actors(self):\n",
        "        movie_info = self._movie_info\n",
        "        actors_list = []\n",
        "        for item in tqdm(movie_info):\n",
        "            actor = item['actor']\n",
        "            actors_list += actor\n",
        "        unique_actors = set(actors_list)\n",
        "        ordered_actors = sorted(list(unique_actors))\n",
        "        ids = list(range(1, len(ordered_actors) + 1))\n",
        "        actors = pd.DataFrame()\n",
        "        actors['id'] = ids\n",
        "        actors['name'] = ordered_actors\n",
        "        self._actors = actors\n",
        "        return actors\n",
        "    def get_casting(self):\n",
        "        movie_info = self._movie_info\n",
        "        actors = self._actors\n",
        "        casting = pd.DataFrame()\n",
        "        for item in tqdm(movie_info):\n",
        "            movie_id = item['id']\n",
        "            names = item['actor']\n",
        "            ords = item['ord']\n",
        "            df = pd.DataFrame()\n",
        "            df['actor_name'] = names\n",
        "            df['ord'] = ords\n",
        "            df['movie_id'] = movie_id\n",
        "            casting = casting.append(df)\n",
        "        casting_merged = pd.merge(casting, actors, left_on='actor_name', right_on='name', how='left')\n",
        "        casting_selected = casting_merged[['movie_id', 'id', 'ord']]\n",
        "        casting_selected.columns = ['movie_id', 'actor_id', 'ord']\n",
        "        return casting_selected"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y77Bb8Ug4zlu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "909b6785-648f-46d8-f3dd-2a4faf27ff01"
      },
      "source": [
        "imdb_scraper = IMDBScraper()\n",
        "top_rated_movies = imdb_scraper.get_top_rated_movies()\n",
        "actors = imdb_scraper.get_actors()\n",
        "casting = imdb_scraper.get_casting()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "250it [03:54,  1.07it/s]\n",
            "100%|██████████| 250/250 [00:00<00:00, 181007.42it/s]\n",
            "100%|██████████| 250/250 [00:00<00:00, 319.27it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZfMD7mKc0i6f"
      },
      "source": [
        "conn = sqlite3.connect('imdb.db')\n",
        "top_rated_movies.to_sql('movies', conn, index=False)\n",
        "actors.to_sql('actors', conn, index=False)\n",
        "casting.to_sql('casting', conn, index=False)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0zHn8UYy4zlv"
      },
      "source": [
        "#top_rated_movies.to_csv('top_rated_movies.csv', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JjMbGApN4zlv"
      },
      "source": [
        "#top_rated_movies.to_json('imdb_top_rated.json', orient='records', force_ascii=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xGOKdD4W4zlv"
      },
      "source": [
        "#top_rated_movies.to_excel('imdb_top_rated.xlsx', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X3nwzQ1e3RU7"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}