exploratory.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from matplotlib import pyplot as plt\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Predicting article popularity on Facebook, Google+, and LinkedIn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "source": [
    "### Download data\n",
    "Source: http://archive.ics.uci.edu/ml/datasets/News+Popularity+in+Multiple+Social+Media+Platforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "wget -nc -O news-final.csv http://archive.ics.uci.edu/ml/machine-learning-databases/00432/Data/News_Final.csv; \n",
    "wget -nc -O facebook-microsoft.csv http://archive.ics.uci.edu/ml/machine-learning-databases/00432/Data/Facebook_Microsoft.csv;\n",
    "wget -nc -O googleplus-microsoft.csv http://archive.ics.uci.edu/ml/machine-learning-databases/00432/Data/GooglePlus_Microsoft.csv;\n",
    "wget -nc -O linkedin-microsoft.csv http://archive.ics.uci.edu/ml/machine-learning-databases/00432/Data/LinkedIn_Microsoft.csv &\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "source": [
    "### Load data and set index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "news = pd.read_csv('news-final.csv')\n",
    "news = news[news['Topic']=='microsoft']\n",
    "news = news.drop('Topic', 1)\n",
    "\n",
    "news['IDLink'] = news['IDLink'].astype(int)\n",
    "news = news.set_index('IDLink')\n",
    "\n",
    "\n",
    "facebook = pd.read_csv('facebook-microsoft.csv')\n",
    "facebook = facebook.set_index('IDLink')\n",
    "\n",
    "google = pd.read_csv('googleplus-microsoft.csv')\n",
    "google = google.set_index('IDLink')\n",
    "\n",
    "linkedin = pd.read_csv('linkedin-microsoft.csv')\n",
    "linkedin = linkedin.set_index('IDLink')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "source": [
    "### Restrict to articles appearing on all 3 sites"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "posted_everywhere = news[['Facebook', 'GooglePlus', 'LinkedIn']].min(axis=1) > -1\n",
    "\n",
    "news = news[posted_everywhere]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Restrict to sources with at least 50 articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "greater_than_50 = (news['Source'].value_counts() > 50)\n",
    "\n",
    "sources_50 = greater_than_50[greater_than_50 > 0].index\n",
    "\n",
    "print(\"Number of sources with at least 50 articles: {}\".format(greater_than_50.sum()))\n",
    "\n",
    "news = news[news['Source'].isin(sources_50)]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "source": [
    "### Process dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "news.loc[:, 'PublishDate'] = pd.to_datetime(news['PublishDate'])\n",
    "\n",
    "news.loc[:, 'Year'] = news['PublishDate'].dt.year\n",
    "news.loc[:, 'Month'] = news['PublishDate'].dt.month\n",
    "news.loc[:, 'DayOfWeek'] = news['PublishDate'].dt.weekday\n",
    "news.loc[:, 'Hour'] = news['PublishDate'].dt.hour"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "news['PublishDate'].max()\n",
    "\n",
    "may_1_2016 = datetime.datetime(2016, 5, 1)\n",
    "\n",
    "news_historical = news[news['PublishDate'] <= may_1_2016]\n",
    "news_new = news[news['PublishDate'] > may_1_2016]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "news_new.to_csv('news-new.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Descriptive Analytics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "news.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**Year published**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "news['PublishDate'].dt.year.value_counts().plot.bar()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**Day of week**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "news['PublishDate'].dt.weekday.value_counts().plot.bar()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**Sentiment**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "news['ones'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "news.plot.scatter('SentimentTitle', 'ones', alpha=0.1, figsize=(14, 0.5))\n",
    "\n",
    "plt.axvline(news['SentimentTitle'].mean(), alpha = 0.4, ls='--')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "news.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "news.plot.scatter(x='SentimentTitle', y='Facebook')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## ML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression, Ridge\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn import base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "class ColumnSelectTransformer(base.BaseEstimator, base.TransformerMixin):\n",
    "    \n",
    "    def __init__(self, col_names):\n",
    "        self.col_names = col_names  # We will need these in transform()\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        # This transformer doesn't need to learn anything about the data,\n",
    "        # so it can just return self without any further processing\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X):\n",
    "        # Return an array with the same number of rows as X and one\n",
    "        # column for each in self.col_names\n",
    "        return X[self.col_names].values  #REMOVE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "df = news_historical.drop(['PublishDate', 'Headline'], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "\n",
    "bag_of_words_vectorizer = CountVectorizer(min_df = 0.05, max_df = 0.95)\n",
    "\n",
    "counts = bag_of_words_vectorizer.fit_transform(df.head()['Title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = bag_of_words_vectorizer.get_feature_names()\n",
    "\n",
    "pd.DataFrame(counts.toarray(), columns=cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import FunctionTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "source_pipe = Pipeline([\n",
    "    ('cst', ColumnSelectTransformer(['Source'])),\n",
    "    ('oh', OneHotEncoder())\n",
    "]) \n",
    "\n",
    "\n",
    "counts_pipe = Pipeline([\n",
    "    ('cst', ColumnSelectTransformer(['Title'])),\n",
    "    ('fn', FunctionTransformer(lambda x: x.reshape(-1))),\n",
    "    ('counts', TfidfVectorizer(min_df = 0.01, max_df = 0.9))\n",
    "])\n",
    "\n",
    "hour_pipe = Pipeline([\n",
    "    ('cst', ColumnSelectTransformer(['Hour'])),\n",
    "    ('oh', OneHotEncoder())\n",
    "]) \n",
    "\n",
    "sentiment_pipe = Pipeline([\n",
    "    ('cst', ColumnSelectTransformer(['SentimentTitle', 'SentimentHeadline']))\n",
    "])\n",
    "\n",
    "features = FeatureUnion([\n",
    "    ('source', source_pipe),\n",
    "    ('hour', hour_pipe),\n",
    "    ('counts', counts_pipe),\n",
    "    ('sentiment', sentiment_pipe)\n",
    "])\n",
    "\n",
    "\n",
    "features.fit_transform(df)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "facebook_model = Pipeline([\n",
    "    ('features',counts_pipe),\n",
    "    ('model', LinearRegression())\n",
    "])\n",
    "\n",
    "facebook_model.fit(df, df['Facebook'])\n",
    "\n",
    "google_model = Pipeline([\n",
    "    ('features', features),\n",
    "    ('model', LinearRegression())\n",
    "])\n",
    "\n",
    "google_model.fit(df, df['GooglePlus'])\n",
    "\n",
    "linkedin_model = Pipeline([\n",
    "    ('features', features),\n",
    "    ('model', LinearRegression())\n",
    "])\n",
    "\n",
    "linkedin_model.fit(df, df['LinkedIn'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "## Validate models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "df_new = news_new.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_features = list(zip(facebook_model.named_steps['features'].named_steps['counts'].get_feature_names(), facebook_model.named_steps['model'].coef_))\n",
    "\n",
    "sorted(word_features, key=lambda x: x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "print('Facebook Model')\n",
    "print('training: {}'.format(facebook_model.score(df, df['Facebook'])))\n",
    "print('testing: {}'.format(facebook_model.score(df_new, df_new['Facebook'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "print('Google Model')\n",
    "print('training: {}'.format(google_model.score(df, df['GooglePlus'])))\n",
    "print('testing: {}'.format(google_model.score(df_new, df_new['GooglePlus'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "print('LinkedIn Model')\n",
    "print('training: {}'.format(linkedin_model.score(df, df['LinkedIn'])))\n",
    "print('testing: {}'.format(linkedin_model.score(df_new, df_new['LinkedIn'])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "facebook_predict = facebook_model.predict(df_new)\n",
    "\n",
    "google_predict = google_model.predict(df_new)\n",
    "\n",
    "linkedin_predict = linkedin_model.predict(df_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new['FacebookPrediction'] = facebook_predict\n",
    "\n",
    "df_new['GooglePrediction'] = google_predict\n",
    "\n",
    "df_new['LinkedInPrediction'] = linkedin_predict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare predicted resutls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "df_new[['Title', 'Source', 'PublishDate', 'FacebookPrediction', 'GooglePrediction', 'LinkedInPrediction']]\\\n",
    ".head()"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}