analyze.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sqlite3\n",
    "from bs4 import BeautifulSoup\n",
    "from datetime import datetime, timedelta\n",
    "from tqdm import tqdm\n",
    "import pickle as pkl\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "from userinfo import YOUR_NAME, START_DAY, END_DAY\n",
    "\n",
    "%matplotlib inline\n",
    "\"\"\"GOALS\"\"\"\n",
    "# Cumulative Messaging per person\n",
    "# Number of people messaged per day\n",
    "# Total Sent/Received"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Num_People_To_Graph = 10\n",
    "\"\"\"Easy way to overwrite your userinfo\"\"\"\n",
    "START_DAY = \"8/1/16\"\n",
    "END_DAY = \"5/7/19\"\n",
    "print(YOUR_NAME, START_DAY, END_DAY)\n",
    "if(not os.path.exists(\"graphs\")):\n",
    "    os.mkdir(\"graphs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "convos = pkl.load(open(\"messages.pkl\", 'rb'))\n",
    "#0: id (string)\n",
    "#1: name (string)\n",
    "#2: num_messages (dict name:num_messages)\n",
    "#3: datetime (list of datetime objs)\n",
    "#4: text (list of messages)\n",
    "#5: who (list of names)\n",
    "#6: group (bool)\n",
    "print(\"Total Conversations: \", len(convos), \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_day = datetime.strptime(START_DAY, \"%m/%d/%y\")\n",
    "end_day = datetime.strptime(END_DAY, \"%m/%d/%y\")\n",
    "num_days = (end_day - start_day).days\n",
    "num_days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mpd = pd.DataFrame()\n",
    "df_mpd_me = pd.DataFrame()\n",
    "\n",
    "date_list = [start_day.date() + timedelta(days=x) for x in range(0, num_days+1)]\n",
    "\n",
    "for i in range(len(convos)):\n",
    "    messages_per_day = []\n",
    "    messages_per_day_me = []\n",
    "    if(convos[i][6] == False):# not group chat\n",
    "        num_messages = len(convos[i][3]) \n",
    "        \n",
    "        total_messages = 0\n",
    "        total_messages_me = 0\n",
    "        prev_total_messages = 0\n",
    "        prev_total_messages_me = 0\n",
    "        for day in date_list:\n",
    "            while(total_messages < num_messages and convos[i][3][num_messages - total_messages - 1].date() < day):\n",
    "                total_messages+=1 #messages until day\n",
    "                if(convos[i][5][num_messages - total_messages - 1] == YOUR_NAME):\n",
    "                    total_messages_me += 1\n",
    "                \n",
    "            messages_per_day.append(total_messages - prev_total_messages) #num messages per day\n",
    "            messages_per_day_me.append(total_messages_me - prev_total_messages_me) #num messages by me per day\n",
    "            prev_total_messages = total_messages #update total\n",
    "            prev_total_messages_me = total_messages_me #update total\n",
    "        if(sum(messages_per_day[1:]) > 0): #no need to add columns with no messages\n",
    "            df_mpd[convos[i][1]] = messages_per_day[1:]\n",
    "            df_mpd_me[convos[i][1]] = messages_per_day_me[1:]\n",
    "\n",
    "df_mpd.index = date_list[:-1]\n",
    "df_mpd_me.index = date_list[:-1]\n",
    "\n",
    "df_mpd = df_mpd.reindex(sorted(df_mpd.columns), axis=1) #sort alphabetically, later sort by total messages\n",
    "df_mpd_me = df_mpd_me.reindex(sorted(df_mpd_me.columns), axis = 1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mpd.head()#messages per day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mpd_me.head()# messages by me per day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook\n",
    "%matplotlib notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sort by total num messages\n",
    "df_mpd = df_mpd[df_mpd.sum().sort_values(ascending = False).index]\n",
    "df_mpd.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#messages per person per day\n",
    "df_mpp_pd = df_mpd.copy().cumsum(axis=0)\n",
    "df_mpp_pd = df_mpp_pd.iloc[:,:Num_People_To_Graph]\n",
    "df_mpp_pd.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt_mpp_pd = df_mpp_pd.plot(title = \"Cumulative Messages Per Person Per Day: \"+START_DAY+\"-\"+END_DAY, figsize=(12,12))\n",
    "plt_mpp_pd.set_ylabel(\"Total Messages\")\n",
    "plt_mpp_pd.set_xlabel(\"Date\")\n",
    "plt_mpp_pd.grid(True, linestyle='--')\n",
    "plt_mpp_pd.axhline(y=0, color='k')\n",
    "\n",
    "plt_mpp_pd.get_figure().savefig(\"graphs/cumulative_messaging_by_day.png\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#total messages per day\n",
    "df_tm_pd = df_mpd.iloc[:].sum(axis = 1)\n",
    "df_tm_pd.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.dates as mdates\n",
    "\n",
    "plt_tm_pd = plt.figure()\n",
    "\n",
    "plt_tm_pd = df_tm_pd.plot(title = \"Total Messages Per Day\", figsize=(12,5))\n",
    "plt_tm_pd.set_ylabel(\"Total Messages\")\n",
    "plt_tm_pd.set_xlabel(\"Date\")\n",
    "plt_tm_pd.grid(True, linestyle='--')\n",
    "plt_tm_pd.axhline(y=0, color='k')\n",
    "plt_tm_pd.axhline(y=np.mean(df_tm_pd), color='g', linestyle='--')\n",
    "\n",
    "plt_tm_pd.get_figure().savefig(\"graphs/total_messages_per_day.png\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#num people messaged per day\n",
    "df_ppl_pd = (df_mpd != 0).astype(int).sum(axis = 1)\n",
    "df_ppl_pd.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt_ppl_pd = plt.figure()\n",
    "plt_ppl_pd = df_ppl_pd.plot(title = \"Number of People Talked to\", figsize=(12,5))\n",
    "plt_ppl_pd.set_ylabel(\"Number of People\")\n",
    "plt_ppl_pd.set_xlabel(\"Date\")\n",
    "plt_ppl_pd.grid(True, linestyle='--')\n",
    "plt_ppl_pd.axhline(y=0, color='k')\n",
    "plt_ppl_pd.axhline(y=np.mean(df_ppl_pd), color='g', linestyle = '--')\n",
    "\n",
    "plt_ppl_pd.get_figure().savefig(\"graphs/num_people_talked_per_day.png\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mpd = df_mpd.reindex(sorted(df_mpd.columns), axis=1)#sort alphabetically again\n",
    "df_mpd.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_total_me = pd.DataFrame()\n",
    "\n",
    "df_total_me['sent'] = df_mpd_me.sum()\n",
    "df_total_me['received'] = df_mpd.sum() - df_total_me['sent']\n",
    "\n",
    "df_total_me = df_total_me.reindex(df_total_me.sum(axis = 1).sort_values(ascending = False).index)\n",
    "df_total_me[:Num_People_To_Graph+10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt_total_me = df_total_me[:Num_People_To_Graph+10].plot.bar(title=\"Total Messages Sent/Received (Top \"+ str(Num_People_To_Graph+10) + \" Most Talked To)\", stacked=True, figsize=(12, 8))\n",
    "plt_total_me.set_ylabel(\"Number of Messages\")\n",
    "plt_total_me.set_xlabel(\"Person\")\n",
    "plt.tight_layout()\n",
    "\n",
    "plt_total_me.get_figure().savefig(\"graphs/total_sent_received.png\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Made by your boy Armaan, if you got any cool ideas hit him up...or Fork/Pull Request --> you know the deal"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}