reddit_tudfe.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reddit - The Ultimate Dataset for Everything\n",
    "## *scroll down for details*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Module installation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install praw psaw wordcloud fontTools nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "from collections import Counter\n",
    "import json\n",
    "import praw\n",
    "from psaw import PushshiftAPI\n",
    "from datetime import date\n",
    "import os\n",
    "import string\n",
    "import datetime\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "import fontTools\n",
    "\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords \n",
    "from nltk.tokenize import word_tokenize  \n",
    "stop_words = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PRAW configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# An example config file has been attached. \n",
    "# For client_id, client_secret and user_agent please refer to the related posts:\n",
    "# https://www.reddit.com/r/redditdev/comments/hasnnc/where_do_i_find_the_reddit_client_id_and_secret/\n",
    "def load_praw_config(path = \"../praw_config.json\"):\n",
    "    with open(path) as json_file:\n",
    "        return json.load(json_file)\n",
    "\n",
    "configuration = load_praw_config()\n",
    "\n",
    "reddit = praw.Reddit(client_id = configuration[\"client_id\"], client_secret = configuration[\"client_secret\"], user_agent = configuration[\"user_agent\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_x(submission, from_pushshift = False):\n",
    "    if from_pushshift:\n",
    "        submission_id = submission[\"id\"]\n",
    "        submission = reddit.submission(submission_id)\n",
    "    \n",
    "    submission.title\n",
    "    submission = vars(submission)\n",
    "\n",
    "    score = submission.pop(\"score\")\n",
    "    \n",
    "    return submission, score\n",
    "\n",
    "def dataset_iterator(subreddit_name='politics', past_dataset_size = 100, before_time=None):\n",
    "    api = PushshiftAPI()\n",
    "    \n",
    "    if not before_time:\n",
    "        before_time = date.today() \n",
    "        \n",
    "    before_time_timestamp = int(before_time.timestamp())\n",
    "\n",
    "    iterator = api.search_submissions(subreddit = subreddit_name, limit = past_dataset_size, before = before_time_timestamp)\n",
    "    \n",
    "    for x in iterator:\n",
    "        x, y = process_x(x.d_, from_pushshift = True)\n",
    "        yield x, y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Wordcloud generation and text processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    text = text.replace(\"-\\r\\n\", \"\")\n",
    "    text = text.replace(\"-\\n\", \"\")\n",
    "    text = text.replace(\"deleted\", \"\")\n",
    "    text = text.replace(\"removed\", \"\")\n",
    "\n",
    "    exclist = string.punctuation.replace(\"-\", \"\") + string.digits + \"∗\"\n",
    "    table_ = str.maketrans('', '', exclist)\n",
    "    text = text.translate(table_)\n",
    "\n",
    "    text = word_tokenize(text)\n",
    "    text = [word.lower() for word in text]\n",
    "    text = [word for word in text if not word in stop_words]  \n",
    "\n",
    "    return \" \".join(text)\n",
    "\n",
    "def create_wordcloud(text_counts, save_to_file = False, filename = \"wordcloud.pdf\"):\n",
    "    wordcloud = WordCloud(font_path = \"arial\", width = 1000, height = 1000, random_state=1, colormap = \"viridis\", mode = \"RGBA\", background_color=None, collocations=False)\n",
    "\n",
    "    wordcloud.generate_from_frequencies(text_counts)\n",
    "\n",
    "    plt.figure(figsize=(10, 10))\n",
    "    plt.imshow(wordcloud) \n",
    "    plt.axis(\"off\")\n",
    "\n",
    "    if save_to_file:\n",
    "        wordcloud_svg = wordcloud.to_svg()\n",
    "        f = open(filename + \".svg\", \"w+\" , encoding=\"utf-8\")\n",
    "        f.write(wordcloud_svg)\n",
    "        f.close()\n",
    "\n",
    "        from svglib.svglib import svg2rlg\n",
    "        from reportlab.graphics import renderPDF\n",
    "\n",
    "        drawing = svg2rlg(filename + \".svg\")\n",
    "        renderPDF.drawToFile(drawing, filename)\n",
    "\n",
    "        os.remove(filename + \".svg\")\n",
    "\n",
    "def get_subbreddit_name(phrase):\n",
    "    subreddit_list = list(praw.models.Subreddits(reddit, _data = None).search(phrase))\n",
    "    return [x.display_name for x in subreddit_list][0] if len(subreddit_list) > 0 else None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Search for subreddit and generate wordclouds!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Specify the search phrase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEARCH_TOPICS = [\"music\", \"rock\", \"The Beatles\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Specify the numbers of posts to be processed \n",
    "### (Each number will produce a different wordcloud!)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LAST_N_POST_COUNT = [200]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (Optionally) Specify the date BEFORE which the posts should be fetched (default: current time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BEFORE_DATE = None\n",
    "# Or simply:\n",
    "BEFORE_DATE = '01/09/20 00:00:00'\n",
    "\n",
    "if not BEFORE_DATE:\n",
    "    BEFORE_DATE = datetime.datetime.now()\n",
    "else:\n",
    "    BEFORE_DATE = datetime.datetime.strptime(BEFORE_DATE, '%d/%m/%y %H:%M:%S')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Wait for posts to be downloaded and wordcloud to generate...\n",
    "### (this depends on how fast Reddit works today)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for topic in tqdm(SEARCH_TOPICS):\n",
    "    print(\"Processing search topic: {}\".format(topic))\n",
    "    subreddit_name = get_subbreddit_name(topic)\n",
    "\n",
    "    posts_text = \"\"\n",
    "    titles_text = \"\"\n",
    "    \n",
    "    for past_dataset_size in LAST_N_POST_COUNT:\n",
    "        print(\"Processing last {} posts from subreddit r/{} (https://www.reddit.com/r/{}/)...\".format(past_dataset_size, subreddit_name, subreddit_name))\n",
    "        for x, y in tqdm(dataset_iterator(subreddit_name = subreddit_name, past_dataset_size = past_dataset_size, before_time = BEFORE_DATE)):\n",
    "            posts_text = posts_text + \" \" + clean_text(x[\"selftext\"])\n",
    "            titles_text = titles_text + \" \" + clean_text(x[\"title\"])\n",
    "\n",
    "        posts_words = [word for word in posts_text.split() if len(word) > 2]\n",
    "        titles_words = [word for word in titles_text.split() if len(word) > 2]\n",
    "        posts_word_counts = Counter(posts_words)\n",
    "        titles_word_counts = Counter(titles_words)\n",
    "\n",
    "        title_wordcloud_file_path = \"{}_last_{}_titles_wordcloud_{}.pdf\".format(subreddit_name, past_dataset_size, BEFORE_DATE.strftime('%d-%m-%y_%H-%M-%S'))\n",
    "        post_wordcloud_file_path = \"{}_last_{}_posts_wordcloud_{}.pdf\".format(subreddit_name, past_dataset_size, BEFORE_DATE.strftime('%d-%m-%y_%H-%M-%S'))\n",
    "        print(\"\\rDone. See files {} and {}\".format(title_wordcloud_file_path, post_wordcloud_file_path))\n",
    "        create_wordcloud(titles_word_counts, save_to_file = True, filename = title_wordcloud_file_path)\n",
    "        create_wordcloud(posts_word_counts, save_to_file = True, filename = post_wordcloud_file_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Enjoy the wordclouds!\n",
    "### (saved in the same folder as this notebook)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Now return to 1. and check if you can find topics that do NOT appear on Reddit *(spoiler: it's hard!)*"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "4f8763616d36299a7e67065bbff61b4594b946871d4c1b06c8acd4b8a7e0d76f"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "metadata": {
   "interpreter": {
    "hash": "4f8763616d36299a7e67065bbff61b4594b946871d4c1b06c8acd4b8a7e0d76f"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}