youtube-captions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Before coding\n",
    "\n",
    "Create a new project\n",
    "\n",
    "https://console.developers.google.com/projectcreate\n",
    "\n",
    "Once you have created the project, enable access to the YouTube Data API\n",
    "\n",
    "https://console.developers.google.com/apis/library\n",
    "\n",
    "Once enabled, it is important that you get credentials for your project\n",
    "\n",
    "https://console.developers.google.com/apis/credentials/wizard?api=youtube.googleapis.com\n",
    "\n",
    "From the options select:  \n",
    "\n",
    "| Option  | Value |\n",
    "| ------------- | ------------- |\n",
    "| ¿Qué API estás usando?  | **YouTube Data API v3**  |\n",
    "| ¿Desde dónde llamarás a la API? | **Servidor Web**  |\n",
    "| ¿A qué tipo de datos accederás? | **Datos públicos**  |  \n",
    "\n",
    "Having selected such values, press: **\"¿Qué credenciales necesito?\"**  and you will be given an alphanumeric string that is your API key, place this value into the `api_key` variable:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "api_key = \"\" # Place your YT api key here\n",
    "assert api_key != \"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now, coding\n",
    "\n",
    "Import the necessary packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import urllib\n",
    "import isodate\n",
    "import os\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.parse import urlencode\n",
    "from slugify import slugify\n",
    "from pytube import YouTube"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "search_url = \"https://www.googleapis.com/youtube/v3/search?\"\n",
    "caption_url = \"https://www.youtube.com/api/timedtext?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "language_preferences = ['es-MX','es']\n",
    "channels = {\n",
    "    'h3h3Productions' : 'UCDWIvJwLJsE4LG1Atne2blQ',\n",
    "}\n",
    "starting_channel = 'UCDWIvJwLJsE4LG1Atne2blQ'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "parameters = {\n",
    "    'key': api_key,\n",
    "    'part': 'snippet',\n",
    "    'type': 'video',\n",
    "    'channelId': starting_channel,\n",
    "    'maxResults': 50,\n",
    "    'order': 'date'\n",
    "}\n",
    "max_pages = 15\n",
    "query_string = urlencode(parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "videos = {}\n",
    "count = 0\n",
    "for channel in channels:\n",
    "    print(\"Searching for\", channel)\n",
    "    parameters['channelId'] = channels[channel]\n",
    "    videos[channel] = []\n",
    "    if 'pageToken' in parameters:\n",
    "        del(parameters['pageToken'])\n",
    "        query_string = urlencode(parameters)\n",
    "    pages = max_pages\n",
    "    page_token = 'FIRST TIME!'\n",
    "    while pages > 0 and len(page_token) > 0:\n",
    "        qurl = search_url + query_string\n",
    "        print(qurl)\n",
    "        r = requests.get(search_url + query_string)\n",
    "        result = json.loads(r.text)\n",
    "        try:\n",
    "            page_token = result[\"nextPageToken\"]\n",
    "        except:\n",
    "            page_token = ''\n",
    "        parameters['pageToken'] = page_token\n",
    "        pages = pages - 1\n",
    "        print(len(result['items']), page_token)\n",
    "        videos[channel].extend(result['items'])\n",
    "        count += len(result['items'])\n",
    "        query_string = urlencode(parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Conversion to dataframes\n",
    "chn = []\n",
    "ids = []\n",
    "pub = []\n",
    "titles = []\n",
    "for c in channels:\n",
    "    for v in videos[c]:\n",
    "        videoId = v['id']['videoId']\n",
    "        publishedDate = v['snippet']['publishedAt']\n",
    "        title =  v['snippet']['title']\n",
    "        chn.append(slugify(c))\n",
    "        ids.append(videoId)\n",
    "        pub.append(publishedDate)\n",
    "        titles.append(title)\n",
    "initial_df = pd.DataFrame({\n",
    "    'channel':chn,\n",
    "    'id': ids,\n",
    "    'published_at': pub,\n",
    "    'title': titles\n",
    "})\n",
    "initial_df['published_at'] = pd.to_datetime(initial_df['published_at'])\n",
    "initial_df.to_csv(\"youtube-captions/h3h3_initial.csv\", encoding='utf-8')\n",
    "print(initial_df.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ids = list(initial_df['id'].values)\n",
    "categories = []\n",
    "default_language = []\n",
    "durations = []\n",
    "license = []\n",
    "viewCounts = []\n",
    "likeCounts = []\n",
    "dislikeCounts = []\n",
    "favoriteCounts = []\n",
    "commentCounts = []\n",
    "\n",
    "batch_size = 50\n",
    "i = 0\n",
    "video_details = \"https://www.googleapis.com/youtube/v3/videos?id=%s&part=snippet,statistics,contentDetails&key=%s\" \n",
    "while i < len(ids):\n",
    "    ids_to_query = ','.join(ids[i:i+batch_size])\n",
    "    q = video_details % (ids_to_query, api_key)\n",
    "    r = requests.get(q)\n",
    "    resultlist = json.loads(r.text)\n",
    "    for result in resultlist['items']:\n",
    "        snippet = result['snippet']\n",
    "        contentDetails = result['contentDetails']\n",
    "        statistics = result['statistics']\n",
    "\n",
    "        categories.append(snippet['categoryId'])\n",
    "        if 'defaultAudioLanguage' in snippet:\n",
    "            default_language.append(snippet['defaultAudioLanguage'])\n",
    "        else:\n",
    "            default_language.append('-')\n",
    "        durations.append(contentDetails['duration'])\n",
    "        license.append(contentDetails['licensedContent'])\n",
    "        viewCounts.append(statistics['viewCount'])\n",
    "        favoriteCounts.append(statistics['favoriteCount'])\n",
    "        likeCount = -1\n",
    "        dislikeCount = -1\n",
    "        commentCount = -1\n",
    "        if 'likeCount' in statistics:\n",
    "            likeCount = int(statistics['likeCount'])\n",
    "            dislikeCount = int(statistics['dislikeCount'])\n",
    "        if 'commentCount' in statistics:\n",
    "            commentCount = int(statistics['commentCount'])\n",
    "        likeCounts.append(likeCount)\n",
    "        dislikeCounts.append(dislikeCount)\n",
    "        commentCounts.append(commentCount)\n",
    "    \n",
    "    i += batch_size\n",
    "\n",
    "details_df = pd.DataFrame({\n",
    "    'id': ids,\n",
    "    'category':categories,\n",
    "    'language': default_language,\n",
    "    'duration': durations,\n",
    "    'license': license,\n",
    "    'views': viewCounts,\n",
    "    'likes': likeCounts,\n",
    "    'dislikes': dislikeCounts,\n",
    "    'favs': favoriteCounts,\n",
    "    'comments': commentCounts\n",
    "})\n",
    "\n",
    "details_df.to_csv(\"youtube-captions/h3h3_details.csv\", encoding='utf-8')\n",
    "print(details_df.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "initial_df = pd.read_csv(\"youtube-captions/h3h3_initial.csv\", index_col=0, parse_dates=['published_at'], na_values=[-1, ''])\n",
    "details_df = pd.read_csv(\"youtube-captions/h3h3_details.csv\", index_col=0, na_values=[-1, ''])\n",
    "\n",
    "\n",
    "initial_df = initial_df.drop_duplicates()\n",
    "details_df = details_df.drop_duplicates()\n",
    "details_df.duration = details_df.duration.apply(lambda iso: isodate.parse_duration(iso).total_seconds())\n",
    "\n",
    "#print(initial_df.info())\n",
    "#print(details_df.info())\n",
    "\n",
    "complete_df = pd.merge(left=initial_df, right=details_df, on='id')\n",
    "complete_df.fillna(-1)\n",
    "complete_df.set_index('id', inplace=True)\n",
    "\n",
    "print(complete_df.describe())\n",
    "complete_df.to_csv(\"youtube-captions/h3h3_complete.csv\", encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vids_subs = []\n",
    "errors = []\n",
    "for index, row in complete_df.iterrows():\n",
    "    videoId = index\n",
    "    title = row['title']\n",
    "    subtitles = ''\n",
    "    i = 0\n",
    "    try:\n",
    "        yt = YouTube('https://www.youtube.com/watch?v=' + videoId)\n",
    "        while len(subtitles) == 0 and i < len(language_preferences):\n",
    "            lang = language_preferences[i]\n",
    "            if yt.captions.get_by_language_code(lang) is not None:\n",
    "                subtitles = yt.captions.get_by_language_code(lang).xml_captions\n",
    "            i = i + 1\n",
    "    except:\n",
    "        print(\"Error\", videoId)\n",
    "        errors.append(videoId)\n",
    "    if len(subtitles) > 0:\n",
    "        vids_subs.append({'id': videoId, 'title': title, 'captions': subtitles })\n",
    "print(\"Done!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(language_preferences)\n",
    "## Getting subs & cleaning them\n",
    "for subs in vids_subs:\n",
    "    soup = BeautifulSoup(subs['captions'], \"lxml\")\n",
    "    texts = soup.find_all('text')\n",
    "    sub_entries = []\n",
    "    for text in texts:\n",
    "        sub_entry = {\n",
    "            'duration': text.get('dur'),\n",
    "            'start': text.get('start'),\n",
    "            'content': BeautifulSoup(text.get_text(), \"lxml\").text\n",
    "        }\n",
    "        sub_entries.append(sub_entry)\n",
    "    del(subs['captions'])\n",
    "    subs['captions_parsed'] = sub_entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from os.path import join\n",
    "## Now saving the good stuff\n",
    "directory = \"youtube-captions\\\\captions\"\n",
    "if not os.path.exists(directory):\n",
    "    os.makedirs(directory)\n",
    "print(\"Saving to\", directory)\n",
    "for vid in vids_subs:\n",
    "    file_path = join(directory, slugify(vid['title']) + '.json')\n",
    "    with open(file_path, 'w') as outfile:\n",
    "        json.dump(vid, outfile, indent=4)\n",
    "print(\"Done!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cleaning_subs = \"\"\"directories = ['amlo', 'presidencia', 'epn']\n",
    "for d in directories:\n",
    "    directory = join('youtube-captions', d)\n",
    "    for file in os.listdir(directory):\n",
    "        if file.endswith(\"json\"):\n",
    "            video = None\n",
    "            file1 = join(directory, file)\n",
    "            with open(file1, 'r') as captions_file:\n",
    "                video = json.load(captions_file)\n",
    "                captions = video['captions_parsed']\n",
    "                for cap in captions:\n",
    "                    try:\n",
    "                        cap['content'] = BeautifulSoup(cap['content'], \"lxml\").get_text()\n",
    "                    except:\n",
    "                        print(\"Error\", file1)\n",
    "            with open(join(directory, file), 'w') as captions_file:\n",
    "                json.dump(video, captions_file)\n",
    "print(\"Done!\")\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}