demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pem import Pem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "# Demo of Pem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/myli/miniconda3/envs/main/lib/python3.7/site-packages/ekphrasis/classes/tokenizer.py:225: FutureWarning: Possible nested set at position 2190\n",
      "  self.tok = re.compile(r\"({})\".format(\"|\".join(pipeline)))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading english - 1grams ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/myli/miniconda3/envs/main/lib/python3.7/site-packages/ekphrasis/classes/exmanager.py:14: FutureWarning: Possible nested set at position 42\n",
      "  regexes = {k.lower(): re.compile(self.expressions[k]) for k, v in\n"
     ]
    }
   ],
   "source": [
    "pem_en = Pem(\n",
    "    liwc_path           ='english_liwc15.csv',\n",
    "    emolex_path         ='english_emolex.csv', \n",
    "    estimator_path      ='english_twitter_politeness_estimator.joblib',\n",
    "    feature_defn_path   ='english_twitter_additional_features.pickle',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "pem_en.load('tweets.csv')\n",
    "pem_en.tokenize()\n",
    "pem_en.vectorize()\n",
    "en_df = pem_en.df[['text']].assign(\n",
    "    score=pem_en.predict_proba(),\n",
    "    label=pem_en.predict(),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading english - 1grams ...\n"
     ]
    }
   ],
   "source": [
    "pem_zh = Pem(\n",
    "    liwc_path           ='chinese_liwc15.csv',\n",
    "    emolex_path         ='chinese_emolex.csv', \n",
    "    estimator_path      ='chinese_weibo_politeness_estimator.joblib',\n",
    "    feature_defn_path   ='chinese_weibo_additional_features.pickle',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "pem_zh.load('weibo.csv')\n",
    "pem_zh.tokenize()\n",
    "pem_zh.vectorize()\n",
    "zh_df = pem_zh.df[['text']].assign(\n",
    "    score=pem_zh.predict_proba(),\n",
    "    label=pem_zh.predict(),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>score</th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "      <th>score</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>thanks so much for this awesome politeness pre...</td>\n",
       "      <td>0.845513</td>\n",
       "      <td>Polite</td>\n",
       "      <td>谢谢 您 ， @tslmy ， 制作 了 这么 棒 的 好 作品 。</td>\n",
       "      <td>0.746791</td>\n",
       "      <td>Polite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RT @tslmy: fuck you. This thing is completely ...</td>\n",
       "      <td>-0.716406</td>\n",
       "      <td>Rude</td>\n",
       "      <td>回复 @tslmy ： 白痴 蠢货 这 啥 玩儿 这是</td>\n",
       "      <td>-0.871539</td>\n",
       "      <td>Rude</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text     score   label  \\\n",
       "0  thanks so much for this awesome politeness pre...  0.845513  Polite   \n",
       "1  RT @tslmy: fuck you. This thing is completely ... -0.716406    Rude   \n",
       "\n",
       "                                 text     score   label  \n",
       "0  谢谢 您 ， @tslmy ， 制作 了 这么 棒 的 好 作品 。  0.746791  Polite  \n",
       "1         回复 @tslmy ： 白痴 蠢货 这 啥 玩儿 这是 -0.871539    Rude  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat([\n",
    "    en_df,\n",
    "    zh_df,\n",
    "], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}