BB_review_aug24_3_topics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import multi_rake\n",
    "import re\n",
    "import string\n",
    "import sklearn\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from multi_rake import Rake\n",
    "import nltk\n",
    "import csv\n",
    "#nltk.download('punkt')\n",
    "#nltk.download('stopwords')\n",
    "#nltk.download('wordnet')\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "from nltk.stem import PorterStemmer \n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n",
    "#nltk.download('averaged_perceptron_tagger')\n",
    "from nltk import word_tokenize, pos_tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>star</th>\n",
       "      <th>sku</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>its a bagit holds candy not included trick or ...</td>\n",
       "      <td>5</td>\n",
       "      <td>8566349</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>the bag would be a great deal if it was availa...</td>\n",
       "      <td>1</td>\n",
       "      <td>8566349</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>loved the movie and it played ok was delivered...</td>\n",
       "      <td>5</td>\n",
       "      <td>6083463</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>as a guy i cant say im headoverheels for dirty...</td>\n",
       "      <td>3</td>\n",
       "      <td>6083463</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>this movie is great for all ages shipping was ...</td>\n",
       "      <td>5</td>\n",
       "      <td>8977805</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>154</td>\n",
       "      <td>i bought this as a christmas gift for a family...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>155</td>\n",
       "      <td>our purchase was as a christmas gift for our g...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>156</td>\n",
       "      <td>this is just a quick note for anyone who was c...</td>\n",
       "      <td>4</td>\n",
       "      <td>3792604</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>157</td>\n",
       "      <td>stargate sg the entire series just amazing to ...</td>\n",
       "      <td>5</td>\n",
       "      <td>8553399</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>158</td>\n",
       "      <td>i was very excited to receive this set however...</td>\n",
       "      <td>1</td>\n",
       "      <td>8553399</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>159 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               content  star      sku  label\n",
       "0    its a bagit holds candy not included trick or ...     5  8566349      1\n",
       "1    the bag would be a great deal if it was availa...     1  8566349      0\n",
       "2    loved the movie and it played ok was delivered...     5  6083463      1\n",
       "3    as a guy i cant say im headoverheels for dirty...     3  6083463      0\n",
       "4    this movie is great for all ages shipping was ...     5  8977805      1\n",
       "..                                                 ...   ...      ...    ...\n",
       "154  i bought this as a christmas gift for a family...     5  3792604      0\n",
       "155  our purchase was as a christmas gift for our g...     5  3792604      1\n",
       "156  this is just a quick note for anyone who was c...     4  3792604      1\n",
       "157  stargate sg the entire series just amazing to ...     5  8553399      0\n",
       "158  i was very excited to receive this set however...     1  8553399      1\n",
       "\n",
       "[159 rows x 4 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BB = pd.read_csv('/Users/jiaruizhang/Desktop/reviews/coding/after sort.csv')\n",
    "BB = BB.sort_index()\n",
    "BB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "product = BB.sku.nunique()\n",
    "occur = BB.groupby('sku').count()\n",
    "occur.label.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import FreqDist\n",
    "import seaborn as sns\n",
    "def clean_text_r3(text):\n",
    "    tokenizer = nltk.tokenize.TreebankWordTokenizer()\n",
    "    tokens = tokenizer.tokenize(text)\n",
    "    stemmer = nltk.stem.WordNetLemmatizer()\n",
    "    text = \" \".join(stemmer.lemmatize(token)for token in tokens)\n",
    "    return text\n",
    "round3 = lambda x: clean_text_r3(x)\n",
    "# function to plot most frequent terms\n",
    "def freq_words(x, terms = 30):\n",
    "    all_words = ' '.join([text for text in x])\n",
    "    all_words = all_words.split()\n",
    "    fdist = FreqDist(all_words)\n",
    "    words_df = pd.DataFrame({'word':list(fdist.keys()), 'count':list(fdist.values())})\n",
    "  # selecting top 20 most frequent words\n",
    "    d = words_df.nlargest(columns=\"count\", n = terms) \n",
    "    plt.figure(figsize=(20,5))\n",
    "    ax = sns.barplot(data=d, x= \"word\", y = \"count\")\n",
    "    ax.set(ylabel = 'Count')\n",
    "    plt.show()\n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABJUAAAFACAYAAAAF/03lAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xu0LGV9J/zvD/AuSpAjIcDMMcob45gE48HRiAmKyyDGARQ0jonIkGGcMEHHGMdcjOQyM/qaFY3G0UE0ojIqAQmIjEq4iFfgcAc1AzEovBg4JohxjFHM8/5RtaXZ7LNP1zm7uvtwPp+19trV1dX9/Hb109VV336qdrXWAgAAAABD7DTvAgAAAADY/giVAAAAABhMqAQAAADAYEIlAAAAAAYTKgEAAAAwmFAJAAAAgMGESgAAAAAMJlQCAAAAYDChEgAAAACD7TLvArbFHnvs0davXz/vMgAAAADuMy6//PKvt9bWbWm57TpUWr9+fTZu3DjvMgAAAADuM6rqK9Ms5/Q3AAAAAAYTKgEAAAAwmFAJAAAAgMGESgAAAAAMJlQCAAAAYDChEgAAAACDCZUAAAAAGEyoBAAAAMBgQiUAAAAABhMqAQAAADCYUAkAAACAwXaZdwFradPb3z+Xdtf9x1+aS7sAAAAA82KkEgAAAACDCZUAAAAAGEyoBAAAAMBgQiUAAAAABhMqAQAAADCYUAkAAACAwYRKAAAAAAwmVAIAAABgsFFDpaq6qaquraqrqmpjP2/3qjqvqm7of/9QP7+q6i1VdWNVXVNVPz1mbQAAAABsvVmMVHp6a23/1tqG/vZrkpzfWtsvyfn97SR5dpL9+p/jkrx9BrUBAAAAsBXmcfrbYUlO6adPSXL4xPz3ts7nk+xWVXvNoT4AAAAAtmDsUKkl+URVXV5Vx/Xz9mytfS1J+t+P7OfvneTmicfe0s+7h6o6rqo2VtXGTZs2jVg6AAAAAJuzy8jP/9TW2q1V9cgk51XVl1ZZtlaY1+41o7WTkpyUJBs2bLjX/QAAAACMb9SRSq21W/vftyc5M8mTkty2dFpb//v2fvFbkuw78fB9ktw6Zn0AAAAAbJ3RQqWqekhV7bo0neRZSa5LcnaSo/vFjk5yVj99dpKX9P8F7slJ7lw6TQ4AAACAxTLm6W97Jjmzqpba+V+ttY9V1WVJTquqY5N8NclR/fLnJjk0yY1Jvp3kmBFrAwAAAGAbjBYqtda+nOSnVpj/d0kOXmF+S3L8WPUAAAAAsHbG/u9vAAAAANwHCZUAAAAAGEyoBAAAAMBgQiUAAAAABhMqAQAAADCYUAkAAACAwYRKAAAAAAwmVAIAAABgMKESAAAAAIMJlQAAAAAYTKgEAAAAwGBCJQAAAAAGEyoBAAAAMJhQCQAAAIDBhEoAAAAADCZUAgAAAGAwoRIAAAAAgwmVAAAAABhMqAQAAADAYEIlAAAAAAYTKgEAAAAwmFAJAAAAgMGESgAAAAAMJlQCAAAAYDChEgAAAACDCZUAAAAAGEyoBAAAAMBgQiUAAAAABhMqAQAAADCYUAkAAACAwYRKAAAAAAwmVAIAAABgMKESAAAAAIMJlQAAAAAYTKgEAAAAwGBCJQAAAAAGEyoBAAAAMJhQCQAAAIDBhEoAAAAADCZUAgAAAGCw0UOlqtq5qq6sqnP624+qqkuq6oaq+lBV3b+f/4D+9o39/evHrg0AAACArTOLkUovT/LFidtvSPKm1tp+Se5Icmw//9gkd7TWHpPkTf1yAAAAACygUUOlqtonyXOSnNzfriTPSHJ6v8gpSQ7vpw/rb6e//+B+eQAAAAAWzNgjld6c5NVJ/rm//Ygk32it3dXfviXJ3v303kluTpL+/jv75QEAAABYMKOFSlX1C0lub61dPjl7hUXbFPdNPu9xVbWxqjZu2rRpDSoFAAAAYKgxRyo9Ncm/qaqbknww3Wlvb06yW1Xt0i+zT5Jb++lbkuybJP39D0/y98uftLV2UmttQ2ttw7p160YsHwAAAIDNGS1Uaq39Zmttn9ba+iS/mOSC1tqLk1yY5Mh+saOTnNVPn93fTn//Ba21e41UAgAAAGD+ZvHf35b7L0leWVU3prtm0rv6+e9K8oh+/iuTvGYOtQEAAAAwhV22vMi2a61dlOSifvrLSZ60wjLfSXLULOoBAAAAYNvMY6QSAAAAANs5oRIAAAAAgwmVAAAAABhMqAQAAADAYEIlAAAAAAYTKgEAAAAwmFAJAAAAgMGESgAAAAAMJlQCAAAAYDChEgAAAACDCZUAAAAAGEyoBAAAAMBgQiUAAAAABhMqAQAAADCYUAkAAACAwYRKAAAAAAwmVAIAAABgMKESAAAAAIMJlQAAAAAYTKgEAAAAwGBCJQAAAAAGEyoBAAAAMJhQCQAAAIDBhEoAAAAADCZUAgAAAGAwoRIAAAAAgwmVAAAAABhMqAQAAADAYEIlAAAAAAYTKgEAAAAwmFAJAAAAgMGESgAAAAAMJlQCAAAAYDChEgAAAACDCZUAAAAAGGyXeRdwX3f7O94yl3Yf+bIT5tIuAAAAsGMwUgkAAACAwYRKAAAAAAwmVAIAAABgMKESAAAAAIMJlQAAAAAYbLRQqaoeWFWXVtXVVXV9Vf1eP/9RVXVJVd1QVR+qqvv38x/Q376xv3/9WLUBAAAAsG3GHKn0T0me0Vr7qST7Jzmkqp6c5A1J3tRa2y/JHUmO7Zc/NskdrbXHJHlTvxwAAAAAC2iqUKmqnjrNvEmt863+5v36n5bkGUlO7+efkuTwfvqw/nb6+w+uqpqmPgAAAABma9qRSm+dct49VNXOVXVVktuTnJfkr5N8o7V2V7/ILUn27qf3TnJzkvT335nkESs853FVtbGqNm7atGnK8gEAAABYS7usdmdVPSXJzyRZV1WvnLjrYUl23tKTt9a+n2T/qtotyZlJfnylxZaaW+W+yec8KclJSbJhw4Z73Q8AAADA+LY0Uun+SR6aLnzadeLnm0mOnLaR1to3klyU5MlJdquqpTBrnyS39tO3JNk3Sfr7H57k76dtAwAAAIDZWXWkUmvtk0k+WVXvaa19ZcgTV9W6JN9rrX2jqh6U5JnpLr59YbpA6oNJjk5yVv+Qs/vbn+vvv6C1ZiQSAAAAwAJaNVSa8ICqOinJ+snHtNaescpj9kpySlXtnG5E1GmttXOq6gtJPlhVf5jkyiTv6pd/V5L3VdWN6UYo/eKgvwQAAACAmZk2VPrzJO9IcnKS70/zgNbaNUmesML8Lyd50grzv5PkqCnrAQAAAGCOpg2V7mqtvX3USgAAAADYbmzpQt1LPlJVv1pVe1XV7ks/o1YGAAAAwMKadqTS0f3v35iY15L86NqWAwAAAMD2YKpQqbX2qLELAQAAAGD7MVWoVFUvWWl+a+29a1sOAAAAANuDaU9/O2Bi+oFJDk5yRRKh0nbo1re9ci7t/sjxfzyXdgEAAIC1N+3pb782ebuqHp7kfaNUBAAAAMDCm/a/vy337ST7rWUhAAAAAGw/pr2m0kfS/be3JNk5yY8nOW2sogAAAABYbNNeU+mPJqbvSvKV1totI9QDAAAAwHZgqtPfWmufTPKlJLsm+aEk3x2zKAAAAAAW21ShUlW9IMmlSY5K8oIkl1TVkWMWBgAAAMDimvb0t99OckBr7fYkqap1Sf4yyeljFQYAAADA4pr2v7/ttBQo9f5uwGMBAAAAuI+ZdqTSx6rq40k+0N9+YZJzxykJAAAAgEW3aqhUVY9Jsmdr7Teq6nlJDkxSST6X5NQZ1AcAAADAAtrSSKU3J/mtJGmtfTjJh5Okqjb09z131OrYYXzpbYfNpd3HHn/WXNoFAACA7d2Wrou0vrV2zfKZrbWNSdaPUhEAAAAAC29LodIDV7nvQWtZCAAAAADbjy2FSpdV1b9fPrOqjk1y+TglAQAAALDotnRNpVckObOqXpy7Q6QNSe6f5IgxC4NF8LmTfmHmbT7luHNm3iYAAAAMtWqo1Fq7LcnPVNXTkzy+n/3R1toFo1cGAAAAwMLa0kilJElr7cIkF45cCwAAAADbiS1dUwkAAAAA7kWoBAAAAMBgQiUAAAAABpvqmkrA4vj4uw6dS7s/f+y5c2kXAACAxWSkEgAAAACDCZUAAAAAGEyoBAAAAMBgQiUAAAAABhMqAQAAADCYUAkAAACAwYRKAAAAAAwmVAIAAABgMKESAAAAAIMJlQAAAAAYTKgEAAAAwGBCJQAAAAAGEyoBAAAAMNhooVJV7VtVF1bVF6vq+qp6eT9/96o6r6pu6H//UD+/quotVXVjVV1TVT89Vm0AAAAAbJsxRyrdleTXW2s/nuTJSY6vqscleU2S81tr+yU5v7+dJM9Osl//c1ySt49YGwAAAADbYLRQqbX2tdbaFf30PyT5YpK9kxyW5JR+sVOSHN5PH5bkva3z+SS7VdVeY9UHAAAAwNabyTWVqmp9kickuSTJnq21ryVd8JTkkf1ieye5eeJht/Tzlj/XcVW1sao2btq0acyyAQAAANiM0UOlqnpokjOSvKK19s3VFl1hXrvXjNZOaq1taK1tWLdu3VqVCQAAAMAAo4ZKVXW/dIHSqa21D/ezb1s6ra3/fXs//5Yk+048fJ8kt45ZHwAAAABbZ8z//lZJ3pXki621P5646+wkR/fTRyc5a2L+S/r/AvfkJHcunSYHAAAAwGLZZcTnfmqSX05ybVVd1c/7rSSvT3JaVR2b5KtJjurvOzfJoUluTPLtJMeMWBsAAAAA22C0UKm19umsfJ2kJDl4heVbkuPHqgcAAACAtTOT//4GAAAAwH2LUAkAAACAwYRKAAAAAAwmVAIAAABgsDH/+xuwgzj9zw6ZS7tHHvOxubQLAACAkUoAAAAAbAWhEgAAAACDCZUAAAAAGEyoBAAAAMBgQiUAAAAABhMqAQAAADCYUAkAAACAwYRKAAAAAAwmVAIAAABgMKESAAAAAIMJlQAAAAAYTKgEAAAAwGBCJQAAAAAGEyoBAAAAMJhQCQAAAIDBhEoAAAAADCZUAgAAAGCwXeZdAMAY/uyUZ82l3WOO/sRc2gUAAJg1I5UAAAAAGEyoBAAAAMBgQiUAAAAABnNNJYAZecupPz+Xdk948cfn0i4AAHDfZqQSAAAAAIMJlQAAAAAYzOlvADu4E0+b/Wl5J77AKXkAALC9M1IJAAAAgMGMVAJg4Rxz5iFzaffPjvjYZu979lnHz7CSu/3vw942l3YBAGBLjFQCAAAAYDChEgAAAACDCZUAAAAAGEyoBAAAAMBgQiUAAAAABhMqAQAAADCYUAkAAACAwYRKAAAAAAy2y1hPXFXvTvILSW5vrT2+n7d7kg8lWZ/kpiQvaK3dUVWV5E+SHJrk20le2lq7YqzaAOC+4NAz/3Au7Z57xO/MpV0AABbLmCOV3pPkkGXzXpPk/NbafknO728nybOT7Nf/HJfk7SPWBQAAAMA2Gi1Uaq1dnOTvl80+LMkp/fQpSQ6fmP/e1vl8kt2qaq+xagMAAABg28z6mkp7tta+liT970f28/dOcvPEcrf08+6lqo6rqo1VtXHTpk2jFgsAAADAyhblQt21wry20oKttZNaaxtaaxvWrVs3clkAAAAArGTWodJtS6e19b9v7+ffkmTfieX2SXLrjGsDAAAAYEqzDpXOTnJ0P310krMm5r+kOk9OcufSaXIAAAAALJ5dxnriqvpAkoOS7FFVtyR5XZLXJzmtqo5N8tUkR/WLn5vk0CQ3Jvl2kmPGqgsAAACAbTdaqNRae9Fm7jp4hWVbkuPHqgUAAACAtTVaqAQA7Hie8+G3zKXdjz7vhLm0CwCwI1uU//4GAAAAwHZEqAQAAADAYE5/AwDu855zxskzb/Ojz/+VmbcJADBLRioBAAAAMJiRSgAAc/ALp586l3bPOfLFc2kXALjvMVIJAAAAgMGESgAAAAAMJlQCAAAAYDChEgAAAACDuVA3AABJkuee/uG5tPuRI583l3YBgG0jVAIAYGEddvrH5tLuWUcestn7jjjj0zOs5G5nPv/AubQLAJsjVAIAgPuAo864ZuZt/vnzf3LV+0848+YZVXJPbzli37m0C7CjESoBAAA7jLededtc2j3+iD3n0i7AmIRKAAAAc/Th078+l3afd+Qec2kXuO8QKgEAAHAPF566aS7tPv3F6+bSLrB1hEoAAABsF648+faZt/mEX3nkzNuE7cVO8y4AAAAAgO2PkUoAAACwlW5689/Opd31r/jhubQLk4RKAAAAcB/yt39041za/eFXPWYu7TI/Tn8DAAAAYDChEgAAAACDOf0NAAAAGNVtb758Lu3u+YonzqXdHYVQCQAAANjh3PaWi+bS7p4nHLTq/be/7SOzKWTCI49/7lY9zulvAAAAAAwmVAIAAABgMKESAAAAAIMJlQAAAAAYTKgEAAAAwGBCJQAAAAAGEyoBAAAAMJhQCQAAAIDBhEoAAAAADCZUAgAAAGAwoRIAAAAAgwmVAAAAABhMqAQAAADAYEIlAAAAAAYTKgEAAAAwmFAJAAAAgMEWKlSqqkOq6q+q6saqes286wEAAABgZQsTKlXVzkneluTZSR6X5EVV9bj5VgUAAADAShYmVErypCQ3tta+3Fr7bpIPJjlszjUBAAAAsIJFCpX2TnLzxO1b+nkAAAAALJhqrc27hiRJVR2V5Odba7/S3/7lJE9qrf3asuWOS3Jcf/PHkvzVGpWwR5Kvr9FzrRU1TUdN01vEutQ0HTVNbxHrUtN01DS9RaxLTdNR0/QWsS41TUdN01vEutQ0nft6Tf+ytbZuSwvtskaNrYVbkuw7cXufJLcuX6i1dlKSk9a68ara2FrbsNbPuy3UNB01TW8R61LTdNQ0vUWsS03TUdP0FrEuNU1HTdNbxLrUNB01TW8R61LTdNTUWaTT3y5Lsl9VPaqq7p/kF5OcPeeaAAAAAFjBwoxUaq3dVVX/KcnHk+yc5N2ttevnXBYAAAAAK1iYUClJWmvnJjl3Ts2v+Sl1a0BN01HT9BaxLjVNR03TW8S61DQdNU1vEetS03TUNL1FrEtN01HT9BaxLjVNR01ZoAt1AwAAALD9WKRrKgEAAACwnRAqAQAAADDYDhMqVdVuVfWr/fRBVXXOvGtaTVV9dt41DFVV35pxe5/tf6+vqn87y7a3B5N9fhFqWO19V1UnV9XjZlvd9qmqTqiqL1bVqXNqf+G2TYtY0xBV9bKqesmIzz/XPsMw27q/UlUvraofGae6pKpeUVUPHuv5B9Sxvqqu28bnOKiqfmatalo0a7GOFs0M+vd2tc6q6tx+m3GPfb5ZH+tU1YlV9aopl90ejsP2r6pDZ9zmb82yvb7NQf39vr7NHKqqfr+qnjnvOlZSVTdV1R799OjH6DtMqJRktyRzPcAeorXmDbsFE+tofRKh0r0tQp+fqobW2q+01r4wg3ruC341yaGttRfPo/FF3DYtYk1DtNbe0Vp774hNTN1nqmqh/oHHDmpbt90vTTLaQXeSVyQZFCpV1c4j1bKtDkqyXW8/dkAvzbj9e7vSWju0tfaNLMY+30Lays+1/ZPMNFRKMvNQaSsclAXYZi7CZ0pV7dxa+93W2l/Ou5ZFsCOFSq9P8uiquirJG5M8tKpOr6ovVdWpVVVJUlVPrKpPVtXlVfXxqtprHsXOetTPRLt/0f/t11fVcUu1VNV/raqrq+rzVbVnP/9RVfW5qrqsqv5gDrUuraPXJ3laVV1VVf951nUst9I6nJMf9PmqemP/c11VXVtVL5x1DVn9fXdRVW2oqp2r6j0TdY72elbVq6vqhH76TVV1QT99cFW9v6reXlUb+9fx98aqYws1vrJfF9f1owPekeRHk5w9r76+9L6rqr2q6uK+f11XVU+bRz3Lajqo70v36mNr2Nb6/rlP7v/uU6vqmVX1maq6oaqeVFW799uBa/pt5k9W1U79t0a7TTzXjVW15+Q3vFX16Kr6WL8N+VRVPXYb653sM7++vK5+mROr6qSq+kSSMcOtlepb3sfXVzeq6p39e+8TVfWgkWv4g6p6+cTt/1pVL19pm1nLvmGvqj+tqpeucUnTbjd/t//8va5//aqqjkyyIcmp/Xtzm9ZdVT2kqj5a3ef/dVX1unQH9BdW1YX9Mi/q19F1VfWGicd+q7pvcS9J8pQaZ/9ql6o6pe/Tp1fVgzfXTnUj9r7QL/vBqlqf5GVJ/nO/rtZ0GzbltuKGqlrXL79Tv03YYy3rSLLz8vdTVf37vu9cXVVn9Ovt4f02aqe+ngdX1c1Vdb+13i5N2tx7vrqRIp/vX68zq+qH1rp/r2KlfjX5rf+G6j5rdhr7Nawt76ss1XWPfb7+4StuO9awtt+uqr+qqr9M8mNJHlRVl07cv76qrumnD+nr+HSS561xHa/tn/u8qvpAVb2qf33+W1V9MsnLq2pd39cv63+e2j/2SVX12aq6sv/9Y1V1/yS/n+SF/fpc833mWnasUFWvT7f+rqrZjyqetr+vz4jbzEnL108/bxafKUvtL22/V1ovv9v346OqO2Y5sn/MAX0furqqLq2qXas7rnlj3+euqar/sFY1Lqt3/seerbUd4ifdaJbr+umDktyZZJ90wdrnkhyY5H5JPptkXb/cC5O8e071fmtO7e7e/35QkuuSPCJJS/Lcfv7/m+R3+umzk7yknz5+1jUvtde/nufMu4+ttg7nVMdkn39+kvOS7JxkzyRfTbLXjGtY8X3X33dRuh3FJyY5b+Lxu41Y25OT/Hk//akkl/bbgNcl+Q8Tr+POfX0/OePX74lJrk3ykCQPTXJ9kickuSnJHvPoU31dS++7X0/y2xPraNcFqGmzfWwN21qf5K4kP9G3cXmSdyepJIcl+Yskb03yun75ZyS5qp/+kyTH9NP/Oslf9tMnJnlVP31+kv0mlrlgDWq+Kckeq9R1Yv93PGjGr9vm+vhdSfbvlzktyS+NXMf6JFf00zsl+etsZpuZZZ83Sf40yUtHqGea7ebuE495X+7+nL4oyYY1quX5Sd45cfvhk9ugdAHTV5OsS7JLkguSHN7f15K8oJ9e8/2rfj21JE/tb787yW9srp0ktyZ5QD+920Tff9WI/WpL24rXJXlFv/yzkpwxUg33eD9lYr8kyR8m+bV++qwkT59Ydyf302u+XZqixmuS/Fw/7/eTvHmt+/eAfvWqZf1+Q5KL+umxX8Mt7avclG77vj79dqNf9qCM+HmYu7ffD07ysCQ39uvpqiQ/2i/zX5L8TpIHJrk5yX59/z8ta7Tf3r8WV6Xb5941yQ19HRcl+R8Ty/2v3L3t/BdJvthPPyzJLv30M5dev3Qj4v50xH620vHWzI/9tqK/n5iRtplTrJ9RP1MGrJdXTyz3niRHJrl/ki8nOWCyXyU5LncfNz8gycYkj5rR+pp8DUfvWzvSSKXlLm2t3dJa++d0G6P16VL2xyc5r7pvCH8n3cZ4R3JCVV2d5PNJ9k33AfDdJEvfzF6ebl0lyVOTfKCfft8Ma1x0K63DeTswyQdaa99vrd2W5JNJDphDHSu97yZ9OcmPVtVbq+qQJN8csZbLkzyxqnZN8k/pdrg2JHlauh23F1TVFUmuTPKvksz6mk8HJjmztfZ/W2vfSvLhvrZFcVmSY6rqxCQ/0Vr7hznXs2RLfWwt/E1r7dq+jeuTnN+6T+1r+/YOTL9NbK1dkOQRVfXwJB9Kt+OTJL/Y3/6BqnpoumHlf95/Bv3PdEHGWtlcXUlydmvtH9ewrWnrWamP/01r7ap+mcnPnFG01m5K8ndV9YR0B4VXZnG2mcnm+/TTq+qSqro2XUj4r0Zo+9okz6yqN1TV01prdy67/4B0BxubWmt3JTk1yc/2930/yRn99Fj7Vze31j7TT78/yc+v0s416Ua4/FK6EGMWtrSteHeSpeup/bskfzZSDcvfT4+vbsTRtUlenLv7zr22UTPYLq1U46PTBX+f7Oedkrv71Sws71cHrrLs2K/hlvZVVjPm5+HT0m2/v91a+2a6L5qTLjB6QT/9wnR96rHpXuMb+v7//jWs48AkZ7XW/rHfD/nIxH2Tn7HPTPKnfR8+O8nD+nX68HR9+7okb8o429GVLNKxwpD+PisrrZ9ZfKZM2tx6+dAKy/5Ykq+11i5LktbaN/vPxGcleUlf4yXpwp4xXuu596cd+doJ/zQx/f1066KSXN9ae8p8Spqvqjoo3Ub3Ka21b1fVRem+Xfhe/yGQ3L2ulrTwA6usw3lb0yHP22Cl990PtNbuqKqfSndgcHy6HZN/N0YhrbXvVdVNSY5J923HNUmenm5n9h/TfSNxQF/TezL713FRXrMVtdYurqqfTfKcJO+rqje2ca8LNK1V+9gIbfzzxO1/7ttb6YC1pTsYeEx1p0ocnm6EwKSdknyjtbb/2pb7Ayv1qaVt+P8dqc3VbK6PL38NRz39rXdyum+mfzjdQeKzNrPcXbnnpQNmsV24V5+uqgcm+R/pRmzc3Ie7a15La+3/VNUT011b5L9Xd4rkpNW2U99prX1/Yrkx9q+W74P8wyrtPCddMPFvkry2qmZx8LjqtqJ/7W6rqmekGwE0xrXyVno/vSfdiLKrqzt986D+/rPTvc67pxuJckG6kYRjbpdWqnG3zS04I8v7Vcs93/s/eK+N/RpuYV/li1t4+NifhysdA3woXUjz4SSttXZDVe2/mWXXwmrboMnPtZ3S7Zvf48uTqnprkgtba0f0p3ddtNYFLreAxwpT9/dZWGX9zOIzZdJK6yVZeX+pVlh+af6vtdY+vpaF3aOBBelPO9JIpX9INyxyNX+VZF1VPSVJqjuPfFaJ9SJ4eJI7+g752HRDblfzmXTfZCXj7AhNa5rXdlaGrsMxTa6Xi9OdG75zf0D7s+mGUM+yhi3qz9/eqbV2RpLXJvnpsQrrXZwuPLo43Td+L0v3bd7D0n1o3FndNcSePXIdm6vt8P4c7ockOSJb/lZyZqrqXya5vbX2ziTvyviv1fbk4vTbxP7D/uv9t1YtyZlJ/jjd0Pu/m3xQ/23v31TVUf1jqw9ZR61rDZ9/a+pZlD5+ZpJD0o28+Xg2v838SpLHVdUD+lFeB49QyzTbzaUdxq/3I0mOHPj4qVTizSXJAAAFfUlEQVT3X7a+3Vp7f5I/Svc+n3z+S5L8XFXtUd2FU1+UblTXcmPtX/2Lpefs2/78Su1Ud52gfVtrFyZ5dbrQ4qFZjP2Hk9N9C37axAHT2HZN8rWqul8m9t/6EYOXpjtV95x+pN7Y26WV3Jnkjrr7mi2/nLv71Sxes+X96tPpTiV5Yj/v+cuWH/s1XHFfZeIL32T2ffniJEdUd/2rXZM8N0laa3+dLsB6be4e0fGlJI+qqkf3t1+0hnV8Oslzq+qB/bbwOZtZ7hNJ/tPSjT7oSrr99v+vn37pxPJjrs/NHSt8r39PztqQ/j6LfjbNsdQsjtlXWi+b86UkP1JVB/T17FrdBeI/nuQ/Lr2uVfX/9Ps7a2khjj13mFCp33n/TD+88Y2bWea76XbM3tAPIbsqC3CF+xn6WLpvQK9J8gfpds5W8/Ikx1fVZek69Lxck+Su6i6MNu8LdQ9dh6NZ1uefkm49XZ3um8dXt9b+dsY1rPi+W2bvJBf1w0Tfk+Q3Rywv6XbO9kryuf4Ul+8k+VRr7ep0p8Bcn27Uwmc2/xTjaK1dkW4dXJruwO3k1tqVs65jFQcluaqqrky3w/En8y1noZyYZEO/HXh9kqMn7vtQuuuFrDR8OukO8I7tP4OuT3ftlVnUNXMr9fEkd8yplu8muTB3HxSemRW2ma21m9Od3nFNulO91vw9OeX+yjeSvDPdaVR/ke501CXvSfKOWpsLGf9Ekkv7bfJvpxtdd1KS/11VF7bWvpZuO31hunV1RWvtrBXqHWv/6otJju779O7prhu2Ujs7J3l/dad7XZnkTf06/Ei6A+NRLzq7BWenC7jGOPVtc16b7j13XrqDoUkrbaPG3C5tztFJ3ti/tvunu65Ssrb9e3OW96u3J/m9JH9SVZ9KF5pMGvs1XHFfZXKBye1G3X2h7tH02+8PpXuPnbGsnqU+dFq/7HfSXVvmo9Vd4Pgra1jHZenW/9XpTqHemC6UXO6E9J9/VfWFdMFc0l0r9r9X1WfSbSeWXJjuC4QxLtS9uWOFk5JcU7O/UPeQ/j6LbeYWj6VmdMy+0npZUV/PC5O8ta/nvHRf/pyc5AtJrug/0/9n1n7E4EIce9Y9Q24AgB1LP5LliiRHtdZumHc97DiqakO6kGuRrpnHAF7D+aqqh7bWvlVVD043guq4PvSCrVLdqZDntNYeP+dSths7zEglAIDlqupx6f5z0fkCJWapql6TbpTH2KNyGYnXcCGc1I+mvCLdf28TKMGMGakEAAAAwGBGKgEAAAAwmFAJAAAAgMGESgAAAAAMJlQCAFgQVXVQVZ0z7zoAAKYhVAIAmJOq2nneNQAAbC2hEgDAVqiqV1fVCf30m6rqgn764Kp6f1W9qKqurarrquoNE4/7VlX9flVdkuQpVXVIVX2pqj6d5Hnz+WsAAIYTKgEAbJ2Lkzytn96Q5KFVdb8kBya5Ickbkjwjyf5JDqiqw/tlH5Lkutbav06yMck7kzy3f64fnl35AADbRqgEALB1Lk/yxKraNck/JflcunDpaUm+keSi1tqm1tpdSU5N8rP9476f5Ix++rFJ/qa1dkNrrSV5/yz/AACAbSFUAgDYCq217yW5KckxST6b5FNJnp7k0Um+uspDv9Na+/7kU41VIwDAmIRKAABb7+Ikr+p/fyrJy5JcleTzSX6uqvboL8b9oiSfXOHxX0ryqKp6dH/7ReOXDACwNoRKAABb71NJ9kryudbabUm+k+RTrbWvJfnNJBcmuTrJFa21s5Y/uLX2nSTHJflof6Hur8yscgCAbVTd6fsAAAAAMD0jlQAAAAAYTKgEAAAAwGBCJQAAAAAGEyoBAAAAMJhQCQAAAIDBhEoAAAAADCZUAgAAAGCw/x+DZ5yRag2HKwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>the</td>\n",
       "      <td>525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51</td>\n",
       "      <td>i</td>\n",
       "      <td>319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>63</td>\n",
       "      <td>and</td>\n",
       "      <td>277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>it</td>\n",
       "      <td>256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>a</td>\n",
       "      <td>247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>to</td>\n",
       "      <td>235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>this</td>\n",
       "      <td>181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>wa</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42</td>\n",
       "      <td>of</td>\n",
       "      <td>135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>is</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>in</td>\n",
       "      <td>125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>62</td>\n",
       "      <td>movie</td>\n",
       "      <td>125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>for</td>\n",
       "      <td>108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>on</td>\n",
       "      <td>89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>you</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56</td>\n",
       "      <td>that</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>store</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>99</td>\n",
       "      <td>best</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>103</td>\n",
       "      <td>my</td>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>182</td>\n",
       "      <td>have</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>not</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>buy</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>with</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>89</td>\n",
       "      <td>dvd</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>great</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>240</td>\n",
       "      <td>at</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59</td>\n",
       "      <td>but</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>267</td>\n",
       "      <td>are</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41</td>\n",
       "      <td>price</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44</td>\n",
       "      <td>all</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      word  count\n",
       "29     the    525\n",
       "51       i    319\n",
       "63     and    277\n",
       "0       it    256\n",
       "1        a    247\n",
       "26      to    235\n",
       "13    this    181\n",
       "35      wa    144\n",
       "42      of    135\n",
       "45      is    129\n",
       "37      in    125\n",
       "62   movie    125\n",
       "14     for    108\n",
       "50      on     89\n",
       "10     you     82\n",
       "56    that     78\n",
       "38   store     68\n",
       "99    best     67\n",
       "103     my     66\n",
       "182   have     65\n",
       "5      not     64\n",
       "100    buy     62\n",
       "47    with     60\n",
       "89     dvd     59\n",
       "33   great     57\n",
       "240     at     53\n",
       "59     but     50\n",
       "267    are     45\n",
       "41   price     43\n",
       "44     all     43"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BB['content'] = pd.DataFrame(BB.content.apply(round3))\n",
    "freq_words(BB['content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>star</th>\n",
       "      <th>sku</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>it a bagit hold candy not included trick or tr...</td>\n",
       "      <td>5</td>\n",
       "      <td>8566349</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>the bag would be a great deal if it wa availab...</td>\n",
       "      <td>1</td>\n",
       "      <td>8566349</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>loved the movie and it played ok wa delivered ...</td>\n",
       "      <td>5</td>\n",
       "      <td>6083463</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>a a guy i cant say im headoverheels for dirty ...</td>\n",
       "      <td>3</td>\n",
       "      <td>6083463</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>this movie is great for all age shipping wa ex...</td>\n",
       "      <td>5</td>\n",
       "      <td>8977805</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>154</td>\n",
       "      <td>i bought this a a christmas gift for a family ...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>155</td>\n",
       "      <td>our purchase wa a a christmas gift for our gra...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>156</td>\n",
       "      <td>this is just a quick note for anyone who wa co...</td>\n",
       "      <td>4</td>\n",
       "      <td>3792604</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>157</td>\n",
       "      <td>stargate sg the entire series just amazing to ...</td>\n",
       "      <td>5</td>\n",
       "      <td>8553399</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>158</td>\n",
       "      <td>i wa very excited to receive this set however ...</td>\n",
       "      <td>1</td>\n",
       "      <td>8553399</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>159 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               content  star      sku  label\n",
       "0    it a bagit hold candy not included trick or tr...     5  8566349      1\n",
       "1    the bag would be a great deal if it wa availab...     1  8566349      0\n",
       "2    loved the movie and it played ok wa delivered ...     5  6083463      1\n",
       "3    a a guy i cant say im headoverheels for dirty ...     3  6083463      0\n",
       "4    this movie is great for all age shipping wa ex...     5  8977805      1\n",
       "..                                                 ...   ...      ...    ...\n",
       "154  i bought this a a christmas gift for a family ...     5  3792604      0\n",
       "155  our purchase wa a a christmas gift for our gra...     5  3792604      1\n",
       "156  this is just a quick note for anyone who wa co...     4  3792604      1\n",
       "157  stargate sg the entire series just amazing to ...     5  8553399      0\n",
       "158  i wa very excited to receive this set however ...     1  8553399      1\n",
       "\n",
       "[159 rows x 4 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABJUAAAFACAYAAAAF/03lAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmYLVV5L/7vK4gQQVE5GsThoEGNMcbEo9GIxkiuwRFUnK4aMEau0Thex2scYpIbjfmJ0UQNQQWDV0UUGRwRQXAABWRSQQiKEgkcjZqoUYOu3x9Vzdk0e3fv6u7d3Yfz+TxPP127dlW9q6ZVa797Ve1qrQUAAAAAhrjBWhcAAAAAgK2PpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADDY9mtdgOXYbbfd2saNG9e6GAAAAADXG2edddZ3WmsbFptuq04qbdy4MWeeeeZaFwMAAADgeqOqLptmOre/AQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADDY9mtdgJWy+a1HrkqcDX/y5FWJAwAAALCe6akEAAAAwGCSSgAAAAAMJqkEAAAAwGCSSgAAAAAMJqkEAAAAwGCSSgAAAAAMNrOkUlW9o6quqqoLRsa9vqourKrzquqYqtp15L2XVdUlVXVRVf3BrMoFAAAAwPLNsqfS4Un2nTfuxCR3a63dPcnXkrwsSarqrkmekOTX+nneUlXbzbBsAAAAACzDzJJKrbVTk/z7vHGfaK1d3b88Pclt+uH9kry3tfbT1trXk1yS5N6zKhsAAAAAy7OWz1T6oyQf7Yf3SPKtkfcu78ddR1UdXFVnVtWZmzdvnnERAQAAABhnTZJKVfXyJFcneffcqDGTtXHzttYOba1taq1t2rBhw6yKCAAAAMACtl/tgFV1YJKHJ9mntTaXOLo8yW1HJrtNkm+vdtkAAAAAmM6q9lSqqn2TvCTJI1trPx5567gkT6iqG1XVnkn2SvKF1SwbAAAAANObWU+lqnpPkgcm2a2qLk/yqnS/9najJCdWVZKc3lp7Rmvty1V1VJKvpLst7lmttZ/PqmwAAAAALM/MkkqttSeOGf32Bab/qyR/NavyAAAAALBy1vLX3wAAAADYSkkqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADDYzJJKVfWOqrqqqi4YGXfzqjqxqi7u/9+sH19V9aaquqSqzquq35pVuQAAAABYvln2VDo8yb7zxr00yUmttb2SnNS/TpKHJNmr/zs4yVtnWC4AAAAAlmlmSaXW2qlJ/n3e6P2SHNEPH5Fk/5Hx72qd05PsWlW7z6psAAAAACzPaj9T6VattSuSpP9/y378Hkm+NTLd5f04AAAAANah9fKg7hozro2dsOrgqjqzqs7cvHnzjIsFAAAAwDirnVS6cu62tv7/Vf34y5PcdmS62yT59rgFtNYOba1taq1t2rBhw0wLCwAAAMB4q51UOi7Jgf3wgUmOHRn/h/2vwN0nyQ/mbpMDAAAAYP3ZflYLrqr3JHlgkt2q6vIkr0ry2iRHVdXTknwzyWP7yT+S5KFJLkny4yRPnVW5AAAAAFi+mSWVWmtPnPDWPmOmbUmeNauyAAAAALCy1suDugEAAADYikgqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg61JUqmqnl9VX66qC6rqPVW1Y1XtWVVnVNXFVfW+qtphLcoGAAAAwOJWPalUVXskeU6STa21uyXZLskTkrwuySGttb2SfC/J01a7bAAAAABMZ61uf9s+yU5VtX2SX0pyRZIHJTm6f/+IJPuvUdkAAAAAWMSqJ5Vaa/+a5G+TfDNdMukHSc5K8v3W2tX9ZJcn2WPc/FV1cFWdWVVnbt68eTWKDAAAAMA8a3H7282S7JdkzyS3TnLjJA8ZM2kbN39r7dDW2qbW2qYNGzbMrqAAAAAATLQWt7/9fpKvt9Y2t9b+O8kHk/xOkl372+GS5DZJvr0GZQMAAABgCmuRVPpmkvtU1S9VVSXZJ8lXkpyc5IB+mgOTHLsGZQMAAABgCmvxTKUz0j2Q++wk5/dlODTJS5K8oKouSXKLJG9f7bIBAAAAMJ3tF59k5bXWXpXkVfNGX5rk3mtQHAAAAAAGWovb3wAAAADYykkqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg02VVKqq+00zDgAAAIBtw7Q9ld485TgAAAAAtgHbL/RmVd03ye8k2VBVLxh56yZJtptlwQAAAABYvxZMKiXZIcnO/XS7jIz/jyQHzKpQAAAAAKxvCyaVWmufTvLpqjq8tXbZKpUJAAAAgHVusZ5Kc25UVYcm2Tg6T2vtQbMoFAAAAADr27RJpfcneVuSw5L8fHbFAQAAAGBrMG1S6erW2ltnWhIAAAAAthrTJpWOr6pnJjkmyU/nRrbW/n0mpdoKbX7bP6xKnA3PeNaqxAEAAABYyLRJpQP7/y8aGdeS3GFliwMAAADA1mCqpFJrbc9ZF4TlueItr1iVOLs/8y9WJQ4AAACwvk2VVKqqPxw3vrX2rpUtDgAAAABbg2lvf7vXyPCOSfZJcnYSSSUAAACAbdC0t789e/R1Vd00yT/PpEQAAAAArHs3WOJ8P06y10oWBAAAAICtx7TPVDo+3a+9Jcl2SX41yVGzKhQAAAAA69u0z1T625Hhq5Nc1lq7fAblAQAAAGArMNXtb621Tye5MMkuSW6W5GezLBQAAAAA69tUSaWqelySLyR5bJLHJTmjqg6YZcEAAAAAWL+mvf3t5Unu1Vq7KkmqakOSTyY5elYFAwAAAGD9mjapdIO5hFLvu1n6L8dxPXTZm/ZflTi3f86HViUOAAAAsLBpk0ofq6qPJ3lP//rxST6y1KBVtWuSw5LcLd2vyv1RkouSvC/JxiTfSPK41tr3lhoDAAAAgNlZsLdRVf1KVd2vtfaiJP+Y5O5JfiPJ55Mcuoy4f5fkY621u/TL+2qSlyY5qbW2V5KT+tcAAAAArEOL3cL2xiT/mSSttQ+21l7QWnt+ul5Kb1xKwKq6SZIHJHl7v9yftda+n2S/JEf0kx2RZHXupwIAAABgsMWSShtba+fNH9laOzPdbWpLcYckm5O8s6q+VFWHVdWNk9yqtXZFv/wrktxyicsHAAAAYMYWSyrtuMB7Oy0x5vZJfivJW1trv5nkRxlwq1tVHVxVZ1bVmZs3b15iEQAAAABYjsWSSl+sqqfPH1lVT0ty1hJjXp7k8tbaGf3ro9Mlma6sqt375e+e5KpxM7fWDm2tbWqtbdqwYcMSiwAAAADAciz262/PS3JMVT0pW5JIm5LskORRSwnYWvu3qvpWVd25tXZRkn2SfKX/OzDJa/v/xy5l+WybznnrI2ce4x5/ctzMYwAAAMDWYsGkUmvtyiS/U1W/l+Ru/egPt9Y+tcy4z07y7qraIcmlSZ6artfUUX0vqG8meewyYwAAAAAwI4v1VEqStNZOTnLySgVtrZ2TrsfTfPusVAwAAAAAZmexZyoBAAAAwHVM1VMJWNhp//Twmce4/9NPmHkMAAAAmJaeSgAAAAAMJqkEAAAAwGCSSgAAAAAMJqkEAAAAwGCSSgAAAAAMJqkEAAAAwGCSSgAAAAAMJqkEAAAAwGCSSgAAAAAMJqkEAAAAwGCSSgAAAAAMJqkEAAAAwGCSSgAAAAAMJqkEAAAAwGCSSgAAAAAMJqkEAAAAwGCSSgAAAAAMJqkEAAAAwGCSSgAAAAAMJqkEAAAAwGCSSgAAAAAMJqkEAAAAwGCSSgAAAAAMJqkEAAAAwGCSSgAAAAAMJqkEAAAAwGCSSgAAAAAMJqkEAAAAwGDbr3UBgOX72NsfOvMY+z7tIzOPAQAAwNZDTyUAAAAABpNUAgAAAGCwNUsqVdV2VfWlqjqhf71nVZ1RVRdX1fuqaoe1KhsAAAAAC1vLZyo9N8lXk9ykf/26JIe01t5bVW9L8rQkb12rwgHTO/qd+848xgFP/djMYwAAADC9NempVFW3SfKwJIf1ryvJg5Ic3U9yRJL916JsAAAAACxurW5/e2OSFyf5Rf/6Fkm+31q7un99eZI9xs1YVQdX1ZlVdebmzZtnX1IAAAAArmPVk0pV9fAkV7XWzhodPWbSNm7+1tqhrbVNrbVNGzZsmEkZAQAAAFjYWjxT6X5JHllVD02yY7pnKr0xya5VtX3fW+k2Sb69BmUDAAAAYAqr3lOptfay1tptWmsbkzwhyadaa09KcnKSA/rJDkxy7GqXDQAAAIDprNUzlcZ5SZIXVNUl6Z6x9PY1Lg8AAAAAE6zF7W/XaK2dkuSUfvjSJPdey/IAAAAAMJ311FMJAAAAgK2EpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg22/1gUAWI4jDn/wqsQ58KBPjB3/liP/YFXiP/PJHx87/nXvXZ34L3nC+PgvPnrfVYn/Nwd8bFXiAAAA09NTCQAAAIDBJJUAAAAAGMztbwBstfY/dnVuv/vQfm6/AwCA+fRUAgAAAGAwSSUAAAAABpNUAgAAAGAwSSUAAAAABpNUAgAAAGAwv/4GAEv0kA89b+YxPrr/G2ceAwAAlkJPJQAAAAAGk1QCAAAAYDBJJQAAAAAGk1QCAAAAYDBJJQAAAAAG8+tvALCVeugxfznzGB951J/NPAYAAFsnPZUAAAAAGExPJQBgSR72wTfOPMaHH/28mccAAGBp9FQCAAAAYDA9lQCArdLDPnDozGN8+DEHzzwGAMDWSk8lAAAAAAaTVAIAAABgMEklAAAAAAbzTCUAgIEe/oF3rUqcEx7zh+PjH/2+1Yl/wONXJQ4AsHVa9Z5KVXXbqjq5qr5aVV+uquf2429eVSdW1cX9/5utdtkAAAAAmM5a3P52dZL/3Vr71ST3SfKsqrprkpcmOam1tleSk/rXAAAAAKxDq55Uaq1d0Vo7ux/+zyRfTbJHkv2SHNFPdkSS/Ve7bAAAAABMZ02fqVRVG5P8ZpIzktyqtXZF0iWequqWE+Y5OMnBSXK7291udQoKAMA1HnH0sasS5/gD9hs7fr+jP74q8Y894A/Gjn/UBz4z89jHPGbvmccAgOVas19/q6qdk3wgyfNaa/8x7XyttUNba5taa5s2bNgwuwICAAAAMNGa9FSqqhumSyi9u7X2wX70lVW1e99LafckV61F2QAAYD074APnzDzG0Y+5x8T3nvTBy2Ye/92Pvv3E9/76mCtmHv9lj9p95jEArg/W4tffKsnbk3y1tfaGkbeOS3JgP3xgktXpVw0AAADAYGvRU+l+SZ6S5Pyqmvua5f8keW2So6rqaUm+meSxa1A2AAAAAKaw6kml1tpnktSEt/dZzbIAAAAMccQHN888xoGP9uxYYOuwZg/qBgAAAGDrtSYP6gYAAGCY44/6zqrEecTjdhs7/tQjZ99LK0ke8GQ9tWBroacSAAAAAIPpqQQAAMC696XDrlqVOL/5x7dclThwfaCnEgAAAACD6akEAAAAi/iXN/3bqsS543N+eez4f3v9ZTOP/csvuv3E9/7tDefPPv4Lfn3ie1f+3ekzj3+r595n4ntXvfmkmce/5bP3mRz/H46bffxnPXLwPHoqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg0kqAQAAADCYpBIAAAAAg627pFJV7VtVF1XVJVX10rUuDwAAAADXta6SSlW1XZJ/SPKQJHdN8sSquuvalgoAAACA+dZVUinJvZNc0lq7tLX2syTvTbLfGpcJAAAAgHnWW1JpjyTfGnl9eT8OAAAAgHWkWmtrXYZrVNVjk/xBa+2P+9dPSXLv1tqzR6Y5OMnB/cs7J7loGSF3S/KdZcy/XOKLv1bxt+V1F1988dU94osv/rYVf1ted/HF35bjb8vrvhLxb99a27DYRNsvI8AsXJ7ktiOvb5Pk26MTtNYOTXLoSgSrqjNba5tWYlnii781xd+W11188cVX94gvvvjbVvxted3FF39bjr8tr/tqxl9vt799McleVbVnVe2Q5AlJjlvjMgEAAAAwz7rqqdRau7qq/jTJx5Nsl+QdrbUvr3GxAAAAAJhnXSWVkqS19pEkH1mlcCtyG5344m+F8bfldRdffPG3zdjiiy/+tht/W1538cXfluNvy+u+avHX1YO6AQAAANg6rLdnKgEAAACwFZBUAgAAAGAwSaUJquoZVfWHK7Cc51XVL61EmQbE3FhVFyxzGQ+sqt9ZyzKspKp6dVW9cMppH1hVJ8y6TNOoqntU1UNXeJmvqarfX8llridV9cP1UoaqunVVHd0PH1RVf7+GZTq8qg7ohw+rqruuVVmmMcs6ZKXq9/Wqqr5RVbutdTmm0Z8Xt17jMuw/zfkw6+vaQstfaDtV1a5V9cx++Jo6Z2sy6Rq90tt8JbdPVZ1SVcv6mebRZUxz3k6apqoeWVUvXUY5PrfUeelU1XOq6qtV9b0h+6I/xv/nLMvWx5l5G2Sl2pfroR031Mj+f/eAea6pu1ewHDPbz6PtyGUsYz21hZddh28NpslFDPmcPImk0gSttbe11t61Aot6XpJBSaWq2m4F4i7XA5MsOanEdVXVUh6Mf48kK5ZUqqrtWmuvbK19cqWWyWSttW+31pZ1AZ6F1toft9a+stblWAtVtf0K1u8s30FJ1jSplGT/JOs6yZqFt9OuSZ6ZrN86Z7mqs+w26/V1+7TWjmutvXYZ82vvLd8zkzy0tXazcftigTbgxiQrmlRaifNlKW3Wbbx9Obf/nzRgnmvq7qVYq/28XqzUdWE9mVEOYHAuYimuFzuiz/Jf2H/7fkFVvbuqfr+qPltVF1fVvavq5lX1oao6r6pOr6q7V9UN+m99dh1Z1iVVdavRjF1V3bGqPlZVZ1XVaVV1lwnluHFVfbiqzu3L8ap0jcCTq+rkfponVtX5/fuvG5n3h32G/4wk962qe1bVp/uYH6+q3Qdulu2r6oh+fY+uql+atMw+u/6Vftr3VtXGJM9I8vyqOqeq7j8w9kJluOZbtqra1GeJb9Dvpw39+Bv0+2FZ37RX1cur6qKq+mSSOyfZqaq+MPL+xqo6rx/etz+GPpPk0UuM94p+GSdW1Xuq6oX9+v3fqvp0kudW1Yaq+kBVfbH/u18/772r6nNV9aX+/52raockr0ny+H4/PH6R+HPnwbht/sp+3R5b187S36uPd25VfaGqdqmq7arq9X35zquq/7XE7fGC/ji/oLos+cbqvsX5p6r6clV9oqp26qed6hwbGL/69bigP+ce349/X430/uq3x2NWar3nlWHst+xV9bCq+nxV7TbpmJhy+VNv43nzjX47/sOq+qv+GDi9qm7Vjx9UrgnH/z36ZZ5XVcdU1c36aSeNv2dfjs8nedYi8ebXt4+vyXXc/PNw0fq9qh7bL/fcqjp1iXGf3m+7c/tt+UuTll1VO1bVO/tj9UtV9Xv9+IOq6oN9GS+uqr9ZbDv0bz27qs7ulze3Tte5Dvbjz6/uG9Oqqu9W34urqv65lvCt87TbqLp6aFOSd1dXx13nWF2qCcfjdfZ1dT1yH5nk9X0Z7rjIosdd1/bp99n5VfWOqrpRX4brXO/64Q19uc6uqn+sqstqy/Vuu5p3/k6xnV6b5I79e++vvs7pj50PVdXxVfX1qvrT6uqML/X7/+b9dIPr36p6cVU9px8+pKo+1Q/vU1VH1gJtnZHhA6rq8DHLnqsHzk5yVpLdk5yd5CnV1Ztn9+u5cz/91NexGqmTq+qMqvq1kbin9LFv3O/HL/bbar/+/Z2qayOdV1XvS7LTyLyLbY8Hjyv7Itt4yLl9zbf/1V3T3tbvy69V1cNHpjm239cXVddGvdZ+qa6n9in9sX1hde3p6t97aD/uM1X1pup7dFfVk/vtfk5/PD+rRuqpPu6bJ0y73Vz8GnMdGmLc9qrJdfOKtjmq6m1J7pDkuKp6/rx98YbqPgO8rqp+t1/3c/pja5d05+/9+3HPHxBz0vX/LenOl9tW1VP7Y+DTSe43Mu+ktuirq+rQqvpEknf1466zz/r99f/1x+FJtaX9Ptq+fG1t+Wzxt/242/fTn9f/v10/fs/+/PhiVf3FvPV8UW05j/98wrb4UL8vv1xVB/fjflhVr+vHf7K6dvYpVXVpVT2yn2biOTHEvP3/kprXnu+n+bWR7XheVe2Va9fdr1+r/VyT68uqqr/v9+OHk9yyHz/3eWOhz93X+VwzZt0Gt4XHrP+k68Ir++Vc0K9rLbD/nlZVh4y8fnpVvWHiDr/2vCtV/12TA1gg1mLXmbdW1Zn9efDn/XvPyXVzEfv22+vcqjppJMRdR86R50yz/tfSWtvq/9Jl+a9O8uvpEmVnJXlHkkqyX5IPJXlzklf10z8oyTn98N8leWo//NtJPtkPvzrJC/vhk5LsNTLNpyaU4zFJ/mnk9U2TfCPJbv3rWyf5ZpINSbZP8qkk+/fvtSSP64dvmORzSTb0rx+f5B0Dt0dLcr/+9TuSvGjSMpN8O8mN+uFd56//MvbJ/DK8cN722JTklH74VUme1w8/OMkHlnlM3DPJ+ekyszdJckkf/5wkd+ineUmSP0uyY5JvJdmrP2aOSnLCwHib+mXvlGSXJBf38U5J8paR6f5fkr374dsl+Wo/fJMk2/fDvz+3/um+nf77FdjmLx6Z7vAkByTZIcmlSe41WoYkByf5s37cjZKcmWTPJW7/GyfZOcmXk/xmuvP0Hv00RyV58pBzbMrYPxw5H09Msl2SW6U793ZP8qgkR/TT7NDv+51WYr3HlGFjkgtG92Uf/7QkN1vomJjBNj48yQH98ClJNvXDLckj+uG/GdkGU5crk4//85L8bj/Na5K8sR+eZvzr57bdhJjj6ttJddwpufZ5+OosUr/323aPfnjXJca9xch0f5nk2ZOWneR/J3lnP3yXdMfrjv1xc2kfZ8cklyW57SLl+cZIrGcmOawfnnQdfFuShyW5W5Ivzi2v3487L+H4H7pvNi31fB94PE7a14enPzcWWe7GXLeO/bN0dcid+nHvypZr2Tcy/nr390le1g/v2y9zt2xpy4w7fydup1y7nhkdPijdtW+XdO2OHyR5Rv/eISPlHFz/JrlPkvf3w6cl+UK6tsur+r9JbZ0fjizjgCSHjzknz0vyuyPb+1/67XNqkhv307wkySsz8Do2b/s8P8mf98O7J/laP/x/R7b7rkm+lq6efUG2HLd37/fVpim2x0vGlX3+fs3I8bKEc/ug9G2FdMfzx9K1h/dKcnm21CVXJLlFunPjgpHYc9esB6Y7Tm7Tz//5JHtnSztpz3669yQ5IcmvJjk+yQ378W9JcmCSS0bK/dF+GeOm/cN+eOx1aIb1zoq1OUbifSPdcTp/X5yQZLv+9fHZUn/snO5YfWCGtzknXf9/keQ+I8f03Hm4Q5LPjpRrUlv01ek+Q+3Uvx67z/r99aR+3Cvnre8BSW6e5KLkml8Z33Vk/Q/sh/8oyYf64eNGjoVnZcvx+OB0P4de6Y7HE5I8YMz2uHn/f+64vkVfxof0449J8ol05+RvZMu176BMOCeWsf8nteffPLLNdujjbczC7ZzV2s+T6stHZ0s7+tZJvt/v341Z/HP3gp9rssS2cB/7F+nq3LHXhdFjoh/+52ypXw7PvLZwv33/JVuO888l+fUp9vlK1n+PmyLeQteZ/5Ut58F2/brdffTY7Ic35Np1+dw8r+7X+0b9dv3uXFmn/dtqu7yN8fXW2vlJUlVfTnJSa61V1fnpDsDbp7vgpLX2qaq6RVXdNMn70lWI70zyhP71NfqM5+8kef9IkvNGE8pwfpK/re5buRNaa6fNS4zeK12jcnO/7HcneUC6k+/nST7QT3fndI37E/v5t0tX6Q3xrdbaZ/vhI5P8nwWWeV66b0A/1Jdlpcwvw0JZz3ckOTbJG9NdaN65zNj3T3JMa+3HSVJVx/Xjj0ryuHTfDjy+/7tLuuPn4n7aI9NVsEPsneTY1tp/9cs4fuS90WPq99Nlgude36S6b6pumuSI/puLlq6SWIpJ2/x9Y6a9c5IrWmtfTJLW2n/0ZX9wkrvXlvumb5quYfr1AeXYO932/1G/zA+m2ydfb62d009zVpKNA8+xIfZO8p7W2s+TXNl/g3OvdBX8m6rrTbBvklNba/+1Quu9mN9LdwF78Nz2zoRjorU4qrocAAAPGElEQVT2n4ssa+ptvMhyfpauoTY3/f9YQrnGHf83TteQ/HQ/zRHp9vFNpxz/z0keskC5r1XfJvleFq43r3MOLHLsfTbJ4VV1VJIPLjHu3arqL9N9MN05yccXWPbe6Rqdaa1dWFWXJblT/95JrbUf9GX+Srrr2bfGlWfkujO33LOypffl3hl/HTwt3bXosiRvTXJwVe2R5N9ba0t5tsXQfbPSxh2PO2Zl6pn5dewr0p1zX+vHHZHuQ9EbFynfo5KktfaxqvreyHtDz9/FnNyfs/9ZVT9I16hNun1092XUv2cluWd//fppum+MN6Wrg47P5LbOgkbrgep6Tf9rkv9K15i+a5LP9uXcIV3CY+h17GvZ4qh0H5hela5d8P5+/IOTPLK2PF9ix3Qfch6Q5E19nPOq7+k8xfY4bkLZFzPk3J7vqNbaL5JcXFWXpmvnJMmJrbXvJtdcM/ZO9wFy1Bdaa5f305yT7hj8YZJLW2tz18P3pGsn7ZPug+8X+7LtlOSqJJdW1X3SJXTvnK7Oe9aEaZPJ16Ehpqp3ZtjmmOT9fTsk6bbDG/pz4oOttcsX6ECxkEnX/8taa6f30/x2rn0evi9brimT2qJJctxc3ZnJ+/cX2XJNPTLXvkYmyX8k+UmSw6rr4TK3b++bLcfsP6dLICZd75rHjIyf69344P7vS/3rndOdx9f0Hu49p6oe1Q/ftp/mZ+mSq0l3bPy0tfbfI58L50xzTgwxqT3/+SQvr6rbpNv3F0+x71drP0+qLx+QLe3ob8/1jOkt9rl7oc81y20LX9ZaO726XpiT6tbfq6oXp+tccPN0CbnjxywrrbUf9ev28Kr6arpkyvnjpp1npeq/0RzAQha6zjwnyeOq66m3fbpk413TfcYfdZ90n3u+3q/7v4+89+HW2k+T/LSqrkr3hfzlU5Qr6YNeX/x0ZPgXI69/kW49rx4zT0t38P1KdV0390/3bfKoGyT5fmvtHosVoLX2taq6Z7pn4Px1dd0KRy1Ue/xk5KJTSb7cWpvYBW4Kbd7r/1xgmQ9LV3E8MskraqQ7+DLNL0NLtx/mbrvc8Zo3WvtWVV1ZVQ9KV0EOuSd52vhJdxF8f18xt75Sv8eEaYdYaN/+aGT4BknuO1KRdzN3XSNPbq09qm9In7LEcozb5vPLcE3YMdPPjX92a+3jY96b1qTtMXqe/jxdpTr1ObYSZWit/aS621D+IF1S8T0j0y93vRdzabou0nfKlkbL2GNiCkO28UL+u/VfU/TTz10XhpRrSa3iMcuY+jycX9+m+3C4UL057hyYeOy11p5RVb+drn48p6ru0Vr77sC4h6froXFuVR2U7hvpscvOwttw/j695tq9wHXnp2OmHxejpWugPyvdB+eXp0t4HJAu2TTYEvbNShu3nitVzwy5Voy93mXYvl7uLYGLtY2WtF36D2ffSPLUdN9unpfug8Id031rfs9Js44M7zjm/fn1wH+NjD+xtfbEa03c3cI59XWsv77OrcO/Vne7593TXQv+18i8j2mtXTRv3vnl37JSC2+Pr48r+2IGntvXmX3C60njR42rbyYds5Wu5+/LrjWy6mnpEnUXpvtQ3KrbgNeZtjfpOjS1aeudqrpJZtPmmOSaa09r7bV9kuWhSU6vpT/UetL+mH+dm1RfTWqLzl/GpP37ioXitNaurqp7p/vQ/YQkf5qud+x8bcLwaPy/bq3944T1SFU9MF1C4r6ttR/37bsdc+1j6pq6r7X2i7r2c4SmOSeG+IuMac+31v5fdbc3PSzJx6vqj9O1CReymvt5XH350AWWvdi1Zex26C23LTxX9knXhR3T9QTa1H++fHXGX29GHZauA8aFmb5jw0rVf6M5gIkWuc78V7oe2fdqrX2vulvLp7nGjprY1pzG9eKZSlM6NX2ioq+AvtNa+4++wjkmyRvSdbX77uhMfQb161X12H7eqqrfGBegul9m+XFr7cgkf5vkt9Ilc+aywmck+d3q7h3dLskTk3x6zKIuSrKhqu7bL/eGS0j03G5u/j7O6eOWWd0Dzm7bWjs5yYuz5Rv10XIv1fwyfCZdF7y5xuZj5k1/WLpvPI6a5uRaxKlJHlXdMxB2SfKIJGmt/Uu6E+UV2fIty4VJ9qwtz9IY1PDrfSbJI6p7LsrO6S4a43wi3cU1SdJ/mEy6jP6/9sMHjUw/dD+M2+aTXJjk1lV1r74su/QX2o8n+ZOqumE//k5VdeMBZUi67b9/dc8buXG2dHO9jiHn2BLK8Pjq7hXfkC5xOvdMrfemq5Tvny29R1ZivRdzWbpv6d41ck5POiYWM/U2XqIh5Rp3/P8oyfdqyzPZnpLk063rcTNu/PeT/KCq9u7HL5hYHlPf/nYG1psLHXtVdcfW2hmttVcm+U66bz+Hxt0l3TfjNxxdnwnLHr1G3SldgudaH2qn3A6/tcDkk66D30rX3Xmv1tql6fbnC7PE42ngNlqJa818447HH2dyPTOkDPPr2E+m63H5K/24p2TLdf0bGX+9+0y6Budcj5qbTRF3oTIueRsus/49Nd1xcmq6Y+UZ6W47PD2T2zpXVtWv9m2PR81f4Jh6YO6Zl6cnud/cdu7rvTtl+dex96Zr+9x05Jvpj6d7blH18/7myPrOnT93S3cL3LTbY1zZFzTw3J7vsdU9n/KO6T68zdUl/6O6Z6vtlO6L1M9OXMK1XZjkDrUlKTf3fKeTkhxQVXPPWrl5Vd0+Xc+V/dPt+/ctMu2KmLbemWGbY5oy3rG1dn5r7XXpPkzfJUs7f6e5/p+R5IHV9Ui9YZLHjrw37fV90j67QbovHpLuIePXamv29e5NW2sfSfeQ4Lnlfy5dkinpzqW5+T47b/ycjyf5o9rynJw95soy4qZJvtcnlO6SrifGEEs9JyYZ256vqjuk6+33pnS9F++exff9au3nSfXlqUme0Lejd0+XwJjWpM81ycq1hSfVrXPJlO/0x86iP87QWjsjXVvsf2bLl82LWYv6b9J15ibp2t0/qO6ZdKM9/UePs8+nuz7vOVeOJZRhrG0pqfTqJJuq66782nT3PM55X5InZ/wtQklXwT2tqs5N131uvwnT/XqSL1TXXfjl6Xo9HZrko1V1cmvtiiQvS3JyknOTnN1aO3b+QlprP0t3Aryuj3lOhv8S21eTHNiv783T3VYxbpnbJTmyuu6KX0pySN+oOz5dUmY5D+qeX4a3JvnzJH9XVaelS+6MOi5dQmu5t76ltXZ2uv15TrouhaOV8Nz+Pqqf9ifpunF/uLqHWV+2hHhfTFf+c9NVJmemey7BfM9JfxxWdxvLM/rxf5Pum8jPptsnc05O1x100Qd198Zt80ll/lm6huGb+2PixHQV8WFJvpLk7OoeaPqPGZit7rf/4emSOGf0y/zeArNMe44NcUy6LP656Z7p8eLW2r/1730iXZLpk/12SFZgvafRf/v9pHQ95u6YycfEYssZuo2HmrpcCxz/B6Z7APJ56RqVr+lnmTT+qUn+oboHdS/2bdX8+vaVWVq9OenYe331DxpOd/E+dwlxX5Fu35yY7kNZFlj2W9I9pPn8dHXUQX035MWMu+5M8upMvg6ekS23Bp2WZI8snJQeUqaFttHhSd5WK/ig7gWOx0n7+r1JXlTdA0UXe1D3/Dr2kHTH7fv7ffeLdM+oSiZf7/48yYOrexD1Q9LdCrjY7a6HZ8J26r8M+2x/PI192Osillr/npaui/3nW2tXprvd5bRF2jovTXcrzKcy+RbIpyb5h3T7rnvgRHdrx0FJ3tNv+9OT3GUFrmNHp/swe9TIuL9Id6vGef28cw8OfmuSnfv4L86WLykW2x5jyz5h3UcNObfnuyhdIu+j6Z6j9ZN+/GfS3V50TrpnnEx1m0/fe+CZST7Wt5OuTPKD1v2K6J8l+US/bicm2b219r102//2rbUv9MsYO+2AdVrMkHpnFm2OaTyv+h9pSHeN+2i6dsrV1T00d6oHdU9z/e/Pw1en+xD5yXS3ysyZ6vq+wD77UZJfq6qz0vVAes28WXdJckI/z6fTPb9sLu5T+/FPSfLcfvxzkzyrqr6YLhkxF/8T6Z618/m+fj06103CfCzdDyicl+5cPT3DLOmcWMCk9vzjk1zQH593SfKu0bq7xjyoe7X2cybXl8eku4Xr/HT137iOEJNM2g5z5V52W3iB68L3k/xTX+4PpXtW5DSOSvLZvv5a1BrVf5OuM+em+xz/5XSPlBlNjo7mIjan+8z7wb4empT7GGzuAWqw5qr7NapDWmtLTWKtqaraubX2w+p+4enUJAf3F4TVir8x3XMX7rZaMWHOWh//MGo9H4/VPc/t5/0tIvdN8tZVvBWH67nqbns4obV29LzxB6W7HeRPx803xXLnzqlKl/S7uLV2yGLzcf1TVT9srS36C4br3XLPCa5fqvtFy0NaayctOjHXcX16phJbsap6aZI/yco8S2mtHFpVd033LekR6+UDDKwSxz/ryXo+Hm+X5KjqbgH7WZKnr3F5YBpPr6oD0z0M90vpejMAbNWqatd0vcHOlVBaOj2VAAAAABhsW3qmEgAAAAArRFIJAAAAgMEklQAAAAAYTFIJAGCdqKoH9r9CAwCw7kkqAQCskarabq3LAACwVJJKAABLUFUvrqrn9MOHVNWn+uF9qurIqnpiVZ1fVRdU1etG5vthVb2mqs5Ict+q2reqLqyqzyR59NqsDQDAcJJKAABLc2qS+/fDm5LsXFU3TLJ3kouTvC7Jg5LcI8m9qmr/ftobJ7mgtfbbSc5M8k9JHtEv65dXr/gAAMsjqQQAsDRnJblnVe2S5KdJPp8uuXT/JN9PckprbXNr7eok707ygH6+nyf5QD98lyRfb61d3FprSY5czRUAAFgOSSUAgCVorf13km8keWqSzyU5LcnvJbljkm8uMOtPWms/H13UrMoIADBLkkoAAEt3apIX9v9PS/KMJOckOT3J71bVbv3DuJ+Y5NNj5r8wyZ5Vdcf+9RNnX2QAgJUhqQQAsHSnJdk9yedba1cm+UmS01prVyR5WZKTk5yb5OzW2rHzZ26t/STJwUk+3D+o+7JVKzkAwDJVd/s+AAAAAExPTyUAAAAABpNUAgAAAGAwSSUAAAAABpNUAgAAAGAwSSUAAAAABpNUAgAAAGAwSSUAAAAABvv/Aamn5TiW9afuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>movie</td>\n",
       "      <td>125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>store</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>69</td>\n",
       "      <td>best</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70</td>\n",
       "      <td>buy</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59</td>\n",
       "      <td>dvd</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>great</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>price</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>one</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>love</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>217</td>\n",
       "      <td>like</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>220</td>\n",
       "      <td>online</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>259</td>\n",
       "      <td>good</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>833</td>\n",
       "      <td>series</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1012</td>\n",
       "      <td>season</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>764</td>\n",
       "      <td>show</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>941</td>\n",
       "      <td>set</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>get</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>109</td>\n",
       "      <td>bought</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>227</td>\n",
       "      <td>time</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>would</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>72</td>\n",
       "      <td>received</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>353</td>\n",
       "      <td>well</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>shipping</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>127</td>\n",
       "      <td>even</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>253</td>\n",
       "      <td>see</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>448</td>\n",
       "      <td>first</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>73</td>\n",
       "      <td>order</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1013</td>\n",
       "      <td>episode</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>amp</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>65</td>\n",
       "      <td>fast</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>66</td>\n",
       "      <td>ordered</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>102</td>\n",
       "      <td>make</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>really</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>287</td>\n",
       "      <td>ever</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>332</td>\n",
       "      <td>watch</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          word  count\n",
       "37       movie    125\n",
       "23       store     68\n",
       "69        best     67\n",
       "70         buy     62\n",
       "59         dvd     59\n",
       "20       great     57\n",
       "26       price     43\n",
       "7          one     41\n",
       "50        love     33\n",
       "217       like     33\n",
       "220     online     33\n",
       "259       good     33\n",
       "833     series     32\n",
       "1012    season     29\n",
       "764       show     26\n",
       "941        set     25\n",
       "6          get     24\n",
       "109     bought     22\n",
       "227       time     22\n",
       "19       would     20\n",
       "72    received     20\n",
       "353       well     20\n",
       "11    shipping     19\n",
       "127       even     19\n",
       "253        see     19\n",
       "448      first     18\n",
       "73       order     17\n",
       "1013   episode     17\n",
       "9          amp     16\n",
       "65        fast     15\n",
       "66     ordered     15\n",
       "102       make     15\n",
       "200     really     15\n",
       "287       ever     15\n",
       "332      watch     15"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stop_words = stopwords.words('english')\n",
    "# function to remove stopwords\n",
    "def remove_stopwords(rev):\n",
    "    rev_new = \" \".join([i for i in rev if i not in stop_words])\n",
    "    return rev_new\n",
    "# remove short words (length < 3)\n",
    "BB['content'] = BB['content'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>2]))\n",
    "# remove stopwords from the text\n",
    "overall_reviews = [remove_stopwords(r.split()) for r in BB['content']]\n",
    "freq_words(overall_reviews, 35)\n",
    "# To further remove noise from the text we can use lemmatization from the spaCy library. \n",
    "# It reduces any given word to its base form thereby reducing multiple forms of a word to a single word."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Coherence Score:  0.20973066880886088\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "import gensim\n",
    "from gensim import corpora\n",
    "# libraries for visualization\n",
    "import pyLDAvis\n",
    "import pyLDAvis.gensim\n",
    "tokenized_overall = pd.Series(overall_reviews).apply(lambda x: x.split())\n",
    "dictionary = corpora.Dictionary(tokenized_overall)\n",
    "doc_term_matrix = [dictionary.doc2bow(rev) for rev in tokenized_overall]\n",
    "# Creating the object for LDA model using gensim library\n",
    "LDA = gensim.models.ldamodel.LdaModel\n",
    "# Build LDA model\n",
    "lda_model = LDA(corpus=doc_term_matrix, id2word=dictionary, num_topics=5, random_state=100,\n",
    "                chunksize=1000, passes=50)\n",
    "from gensim.models import CoherenceModel\n",
    "# Compute Coherence Score\n",
    "coherence_model_lda = CoherenceModel(model=lda_model,corpus=doc_term_matrix, texts=tokenized_overall,coherence='c_v')\n",
    "coherence_lda = coherence_model_lda.get_coherence()\n",
    "print('\\nCoherence Score: ', coherence_lda)\n",
    "import numpy as np\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1600/1600 [28:38<00:00,  1.07s/it]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# supporting function\n",
    "corpus=doc_term_matrix\n",
    "id2word=dictionary\n",
    "def compute_coherence_values(corpus, dictionary, k, a, b):\n",
    "    \n",
    "    lda_model = gensim.models.LdaMulticore(corpus=doc_term_matrix, id2word=dictionary,\n",
    "                                           num_topics=k, \n",
    "                                           random_state=100,\n",
    "                                           chunksize=100,\n",
    "                                           passes=10,\n",
    "                                           alpha=a,\n",
    "                                           eta=b,\n",
    "                                           per_word_topics=True)\n",
    "    \n",
    "    coherence_model_lda = CoherenceModel(model=lda_model,corpus=doc_term_matrix, texts=tokenized_overall,coherence='c_v')\n",
    "    \n",
    "    return coherence_model_lda.get_coherence()\n",
    "grid = {}\n",
    "grid['Validation_Set'] = {}\n",
    "# Topics range\n",
    "min_topics = 2\n",
    "max_topics = 10\n",
    "step_size = 1\n",
    "topics_range = range(min_topics, max_topics, step_size)\n",
    "# Alpha parameter\n",
    "alpha = list(np.arange(0.01, 1, 0.1))\n",
    "# Beta parameter\n",
    "beta = list(np.arange(0.01, 1, 0.1))\n",
    "# Validation sets\n",
    "num_of_docs = len(corpus)\n",
    "corpus_sets = [gensim.utils.ClippedCorpus(corpus, num_of_docs*0.75), \n",
    "               corpus]\n",
    "corpus_title = ['75% Corpus', '100% Corpus']\n",
    "model_results = {'Validation_Set': [],\n",
    "                 'Topics': [],\n",
    "                 'Alpha': [],\n",
    "                 'Beta': [],\n",
    "                 'Coherence': []\n",
    "                }\n",
    "# Can take a long time to run\n",
    "if 1 == 1:\n",
    "    pbar = tqdm.tqdm(total=1600)\n",
    "    # iterate through validation corpuses\n",
    "    for i in range(len(corpus_sets)):\n",
    "        # iterate through number of topics\n",
    "        for k in topics_range:\n",
    "            # iterate through alpha values\n",
    "            for a in alpha:\n",
    "                # iterare through beta values\n",
    "                for b in beta:\n",
    "                    # get the coherence score for the given parameters\n",
    "                    cv = compute_coherence_values(corpus=corpus_sets[i], dictionary=id2word, \n",
    "                                                  k=k, a=a, b=b)\n",
    "                    # Save the model results\n",
    "                    model_results['Validation_Set'].append(corpus_title[i])\n",
    "                    model_results['Topics'].append(k)\n",
    "                    model_results['Alpha'].append(a)\n",
    "                    model_results['Beta'].append(b)\n",
    "                    model_results['Coherence'].append(cv)\n",
    "                    \n",
    "                    pbar.update(1)\n",
    "    pd.DataFrame(model_results).to_csv('lda_tuning_results_overall.csv', index=False)\n",
    "    pbar.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd81fX1x/HXIWwEZYqMMAKIsjGCCsoQFSfYOkDtr1WrtWKXXVipbS2tVqsdgiJttbVVqdoWbUWtSpgKEhQNMiQJe8+wQ8b5/XFv6CVm3ITcmffz8ciDez/fcQ8Z99zvZ5yvuTsiIiKVqRPrAEREJDEoYYiISFiUMEREJCxKGCIiEhYlDBERCYsShoiIhEUJQ0REwqKEISIiYVHCEBGRsNSNdQA1qVWrVt65c+dYhyEiklCWLl26y91bV7ZfUiWMzp07k5mZGeswREQSipmtD2c/dUmJiEhYlDBERCQsShgiIhIWJQwREQmLEoaIiIRFCUNERMKihCEiImFRwhARkbAk1cI9EZHaYNLMLF5cvJEid1LMGD+4I5PH9on46yphiIgkkEkzs/jbog3Hnxe5H38e6aShLikRkQTy4uKNVWqvSRFPGGY22sxWm1m2mU0sY/tdZpZlZsvMbIGZnR1s72xmR4Lty8xsWqRjFRGJZ/uPFlDkXua28tprUkS7pMwsBZgKXAJsApaY2WvuviJktxfcfVpw/2uAx4HRwW057t4/kjGKiMS7PYeO8cyCtfzl/XXl7pNiFvE4Ij2GMQjIdvdcADObAYwBjicMd98fsn8TIPJpUkQkAWzff5Tp83J5YfEGjhYWcXnvtgDMytr2uX3HD+4Y8XginTDaA6Eda5uAwaV3MrMJwL1AfWBkyKYuZvYRsB+Y5O7zIxiriEhc2LjnMNPm5vBy5iaK3BnTrx13j0ijW5umwIkD38k0S6qsa6TPXUG4+1RgqpndBEwCvgxsBVLdfbeZnQPMNLNepa5IMLM7gTsBUlNTazp+EZGoyd5xkCfnZPPqsi2kmHFdegfuuiiN1JaNT9hv8tg+xxNGzkNXRC2+SCeMTUDodVIHYEsF+88AngJw93wgP/h4qZnlAD2AE+6Q5O7TgekA6enp6s4SkYTz6ZY8nszIYdbyrTSoW4evXNCZOy7sSttTG8Y6tBNEOmEsAbqbWRdgMzAOuCl0BzPr7u5rgk+vBNYE21sDe9y9yMy6At2B3AjHKyISNUvX72VqRjazV+2gaYO63D08jduGdKHlKQ1iHVqZIpow3L3QzO4B3gJSgGfc/VMzexDIdPfXgHvMbBRQAOwl0B0FcBHwoJkVAkXAXe6+J5LxiohEmrvzfs5unpidzfu5u2neuB7fu7QHXzq/M6c2qhfr8CoU8ZXe7j4LmFWq7YGQx98q57h/AP+IbHQiItHh7sxetYMpGdl8tGEfbZo2YNKVZ3HT4FQa10+MohuJEaWISIIqKnbeWL6VqRk5rNy6n/anNWLy2N5cd04HGtZLiXV4VaKEISISAQVFxby6bAtPzskmd+churZuwq+v78eY/u2ol5KYVZmUMEREatDRgiJeWbqJaXNz2LT3CGed0YypNw1kdO+2pNSJ/GrsSFLCEBGpAYePFfLC4g1Mn5fLjgP5DEg9jQfH9GLEmW2wKJTtiAYlDBGRk5B3pIDn3lvHMwvXsvdwARekteS3N/bn/LSWSZMoSihhiIhUw+6D+TyzcC3PvbeeA/mFXNyzDRNGdmNgavNYhxYxShgiIlWwLS9QEPDFDwIFAa/ofQZ3j0ijV7tTYx1axClhiIiEYeOewzw1N4dXggUBx/Zvz9eHp9GtzSmxDi1qlDBERCqQveMAT2bk8OrHgYKA16d34K5haXRs0bjyg5OMEoaISBmWb87jyTnZvLF8Gw3rpnDrBZ2546KunN4svgoCRpMShohIiKXr9zBldjYZq3fStEFdJgzvxm1Du9CiSf1YhxZzShgiUuu5O+/l7GZKsCBgiyb1+f5lZ/Kl8zvRrGF8FwSMJiUMEam13J13VwYKAi7buI/TmyVeQcBo0ndERGqdomJnVtZWpmZks2rbATq2aMQvrg0UBGxQN7EKAkaTEoaI1BoFRcXM/GgzT83JIXfXIdJaN+HxG/pxTb921E3QgoDRpIQhIknvaEERL2duZNrcXDbvO8LZZzTjyZsHclmvxC8IGE1KGCKStA7lBwsCzs9l54F8BqaexuSxvRl+Zuukq/MUDUoYIpJ08o4U8JdgQcB9hwsY0q0lvxvXn/O7Jl9BwGhSwhCRpLH7YD5/WrCWv74fKAg46qw23D0iuQsCRpMShogkvK15R44XBMwvLOaKPmcwYXg3zm7XLNahRcSkmVnHH6fdN4vxgzsyeWyfiL+uEoaIJKwNu4MFAZduxB3GDggUBExrnbwFASfNzOJvizYcf17kfvx5pJOGEoaIJJw12w/w5JwcXvt4Cyl1jBvP7cjXLqodBQFfXLyx3HYlDBGRoOWb85gyO5u3VgQKAt42pDN3XNiVNrWoIGCRe5Xaa5IShojEvcx1e5iSkc2c1Ttp2rAu94zoxq1DamdBwBSzMpNDShRmfylhiEhccncWZu/midlrWLx2jwoCBo0f3PGEMYzQ9khTwhCRuOLuvBMsCPhxsCDgj686m/GDOqogIP8b2C5JGilmmiUlIrVLUbHzetZWngwpCPjLa/vwxXPaqyBgKZPH9jmeMHIeuiJqr6uEISIxVVBUzL8+2sy0YEHAbm1O4Tc39uPqvioIGG+UMEQkJo4WFPFS5kaeDhYE7NWuGU8FCwLWUUHAuKSEISJRdSi/kOcXr2f6vLXsOpjPOZ2aqyBgglDCEElAk2Zm8eLijRS5R3XQ82TkHS7gz++t49n3AgUBh3ZrxYQRAzivawsliipSaRARCUssS0NUx66D+fxx/lr+tmg9B/MLGXXW6UwYkcYAFQSsFpUGEZGwxbI0RFVszTvC03NzmbEkUBDwyj5nMGFEN846IzkLAkaLSoOISNhiWRoiHOt3H2La3BxeWboJd7g2WBCwaxIXBIympC4NYmajgd8BKcAf3f3hUtvvAiYARcBB4E53XxHcdh9we3DbN939rUjHKyLV89n2AzyZkc1rH2+hbkodxp2byteGdaVD8+QvCBhNSVsaxMxSgKnAJcAmYImZvVaSEIJecPdpwf2vAR4HRpvZ2cA4oBfQDnjHzHq4e1EkYxaRqsnalMeUjDW89el2GtdP4asXduWrQ7vUqoKA0dS1dWPW7DhUZnukRfoKYxCQ7e65AGY2AxgDHE8Y7r4/ZP8mQEnqHAPMcPd8YK2ZZQfP936EYxaRMCxZt4cps7OZ+9lOmjWsyzdHBgoCNq+FBQGjKbuMZFFRe02KdMJoD4SO0GwCBpfeycwmAPcC9YGRIccuKnVs+zKOvRO4EyA1NbVGghaRsrk789fsYkpGNh+s3UPLJvX5wegz+dJ5nWhaiwsCRlN5IxXRGMGKdMIoq1Ptc/8vd58KTDWzm4BJwJercOx0YDpAenp6fIz6iSSZ4mLnnZXbmZqRzceb8mjbrCEPXHU24wel0qi+6jzVFpFOGJuA0Jq7HYAtFew/A3iqmseK1HrffeljHruhX42dr6jY+c8nW3gyI4fV2w+Q2qIxD32hD18YqIKAtVGkE8YSoLuZdQE2ExjEvil0BzPr7u5rgk+vBEoevwa8YGaPExj07g58EOF4RRLaPz7cRIN6dfjltSc3H/9YYTEzP9rMk3OyWbf7MN3bnMJvb+zPVX3PUEHAGDPK7n6Kxlr5iCYMdy80s3uAtwhMq33G3T81sweBTHd/DbjHzEYBBcBeAt1RBPd7icAAeSEwQTOkRCr3wuINNKhbh59c3avKxx4tKOLvSzby9NwctuQdpXf7Zky7ZSCXnq2CgPHi5vNSy7yB0s3nRX4M1zxOFvvUhPT0dM/MzIx1GCIR1Xni6+Vum/f9EVz0aAYAX7uoK/ddcVZY5zyYX8jzi9bzh/mBgoDpnZpzz8huDOuhgoDx6JLH55wwtbZ7mya8fe/wap/PzJa6e3pl+2mlt0RcTf9yS/lSWzYm43vDGfHrOTw9L5f6devw3UvPLHf/fYePBQoCLlxH3pECLuweKAg4uIsKAsarSTOzPrcOY82OQ0yamRXx0iDqjJSIKp0sIPDLfcnjc2ITUC3QpVUT3rn3IgCemJ3NE++u+dw+Ow/k89AbKxny8Gx++84azu3cgpkThvDX2wdzXteWShZx7PkyuqMqaq9JusKQiCprRWpF7VIzurVpypvfvpDRv53PY29/Rr26dbhrWBpb9h1h+rxcXvxgAwVFxVzZtx0TRqTRs23kCgImYin2eJbM6zBEJEZ6tm3Gf74xlKueWMDDb6zi4TdWUS/FcIcvDGzP14d3o0urJhGNIdFKsUvF1CUlksTqpdQ5ISkUFjtzvj+cR67rF/FkARWX4pbEoysMkaBk6jr5ZNM+pszO5r8rttOkfgrndm7OknV7cYcFa3YxblB0yujEeyn2RHRLOdNqb4nCtFolDBGSp+vkg7V7mJKRzbxgQcBvXdydW4d05rTG9Vmcu5sbpy9i4j+zqJdShy+e0yHi8cSyFHeyKvl9LPn9jOaHG3VJiUCZn9gqao9XNzz9Piu25PHD0T1ZOHEk37mkB6c1DlSPHdy1Jc9/NVD787svf8yryzZHPJ7xgztWqV3CE5occh66ImofapQwRJLIT64+m/k/GMnXh6eVWT12SLdW/OW2QQB8a8Yy3sjaGtF4Jo/tw5j+Zxx/nmLGLeelJtRVWzwKnZbeeeLrUZumroQhkkRuHdKl0uqxw3q05k9fDizq/frzH/L2iu0Rjek7o/63cDCan4aTVSzXNilhiCSIY4XFzPigZrrILj7rdKbdcg4AdzyXScbqHTVyXom8WK5tUsIQiXNHC4r488K1DH80g4n/zKqx847u3ZYpNw0A4NZnl7Bgza4aO7ckJyUMkTh14GgBT83JYeivZvPTf6+gQ/PG/OW2QTStX/afbbMGVb8/xVV92/F48P4Zt/xpMYtyd59UzGX5zTurjz9Ou28Wk2bWXNKrSZNmZpF23yw6T3w9ruOMJU2rFanEezm7uCCtVdReb9/hYzy7cB1/fu9/BQHvGdGNwV1bAnDgWHGZx+3Pr171/y8M7EBhkfODf3zCuOmLeOWu80nv3KLa8YeaNDOLV5f9b2A9XqcrJ9K06roGhWUsY6kbhZnKusIQqcRNf1jMQ2+sjPjr7DhwlIdmBQoC/u7dNQzu0oJXgwUBS5JFpNxwbkcmj+0NwHXT3mfZxn01ct5EWemdSNOqy0oWFbXXJF1hiITh6bm5vJK5iSX3j6rxGwlt3neE6XNzmLFkIwVFxVzVtx13R7ggYFluOa8TBUXF/OzfKxg7dSH/+cZQerc/9aTOqZXeyUVXGCKVWHL/KAB2HzpG1x/NYtfB/Bo579pdh/jBKx8z7JEMXvhgA2P7t+fd7w7n9+MHRD1ZlLh1SBd+dEVPAK56YgErt+6PSRwSn5QwRCrRumkDcn95Bc0bBxbCpU9+h3mf7az2+VZvO8A3X/yIix+bw6vLtnDz4FTmfH8Ev7qub1QKAlbmzovS+P5lgbUTl/9uPmu2H4hxRBIvlDBEwlCnjvHRA5dy50VdAfi/Zz7gZ//+tErn+HjjPu54LpPLfjuPd1du546LurLghyP52ZjetD+tUSTCrrYJI7rxzYu7A3DJb+aRu/NgjCOSeKCEIVIFP7rirOP1mJ5duI6zH3iT4uKK++MX5+7mS39azJipC/lg7R6+Pao7CyeO5L7Lz6J10wbRCLta7r2kB3cNSwNg5GNzWb9bN72q7ZQwRKpoSLdWfPjjSwA4fKyIrj+axfb9R0/Yx92Zs3oH1097jxunL2Ll1v1MvDxQEPDbo/5XEDDeTby8J7cP7QLAsEfnsGnv4RhHJLEUdsIws05mNir4uJGZNY1cWCLxrUWT+qx96ArOOLUhAIN/+S7vrtxOcbHz5vJtXDNlIV95dgmb9h7hZ9f0YsEPR3LXsDROaZB4ExN/fNXZx++1MPRXGWzNOxLjiCRWwvrtNbM7gDuBFkAa0AGYBlwcudBEoqdZg5QyF75VtHrazHj/vov59VurmZKRze1/yTy+rXPLxvzqi324dkAH6tdN/Av5yWP7kF9QzMtLN3H+Q7P54EcX06ZZw0qPM8q+17TuhpGYwv1NngAMAfYDuPsaoE2kghKJtvJWSVe2ejq/sIj2zT8/YP3f7wzjxnNTkyJZlHj0+n6M6d8OgEG/fDes6cXlje5oFUZiCve3Od/dj5U8MbO66GcuCaqwqJh3V27nzucy6TzxdTpPfL3K5zhyrIhnF65l+KNzuO+fWfTtcCqPXNf3+PYek95gy77k67r53bgBjO7VFghML95z6FglR0gyCbdDda6Z/QhoZGaXAHcD/45cWCI1I7+wiLc+3c7fl2xgYfbJF9Y7cLSAvy5az5/mr2X3oWMM6tKCX32xLxd2b4WZcf05HRj52FzW7jrEBQ/PZtotAxnd+4zKT5xApn3pHG778xJmr9rBwJ+/zbIHLkmYQXw5OeEmjInA7UAW8DVgFvDHSAUlUh0H8wv5z8dbmLFkY6W1kNI7NefGcztyZd8zaFy/bqVXGXsPHePZ99bx54Vr2X+0kIt6tOaeEd0Y1OXEIn1mRsb3hvP7d9fw+NufcdffPuQLA9vz+A39T/r/F0+e+cq53PzHRSzM3k3/B9/mk59eSrMy7vAnyaXShGFmKcBf3P0W4A+RD0mkcnsOHeNfH23m70s28Nn2iheVXdSjNePO7cios06v1pjCL2et5G+L1nP4WBGX9TqdCSO60bfDaRUe882LuzO0eyu+8OR7/PPDzfzzw82s+cXl1EtJnjGN5796Htc99R6Z6/fS96f/ZfnPLkvIWWASvkp/uu5eZGatzax+6DiGSLRszTvCK5mbmLFkI5srGRe4vHdbbjy3Ixd2b01KDRUJ/OP8XK7u1467h3fjzLbhzyYfmNqcT356KX1/+l8Aut//BvN/MIKOLRrXSFzx4JWvX8DVTywga3MevX/yFisevIzG9ZU0klW4P9l1wEIzew04vtzT3R+PRFBSe+XuPMjfl2xkxpKN5B0pqHDf687pwI3ndiS9U3PMIjdRc/Z3h9O5mjWemjWsx9qHruCK3wcK+V34SAZPjB/A1f3a1XCUsfPvbwzlksfnsmbHQc5+4C1W/Xw0DetV/WZOEv/CTRhbgl91AC3YkxqxfHMeM5Zs4O9LNlJQVP6ku4b16jDu3FRuSO/I2e2iX8W1usmihJnxxrcu5Om5OTz0xiq+8eJHvPnpNqbeNLCGIoy9t+8dxoWPzGbjniP0/PGbShpJKqyE4e4/Awiu7nZ3VyUyOWlXPbHgc20tm9TnxnM7ckN6x5N+ow7Hso37mDI7O+KvA/C1YWmcn9aSa6Ys5PVPtvL6J6/z2eTLk2atxvwfjGTQL95hx4F8ev74TT6bfHmsQ5IaFu5K797AXwms9MbMdgH/5+5VK9cpEqJji0aMOzeVLw7sQNtTK181XFPcncVr9zBldjYLsndxaqPoze7p2+E0lv/sMnr/5C0gsF5jzveq3+UVbz64fxR9f/oW+48W0mPSG7EOR2pYuB9tpgP3unsnd+8EfBfNmJKTNP8HI5kwolvUkoW7k7F6B9dPe59x0xexatsB7gsWBIymUxrUZd3DV9K/Y2Cm1fBfz+EfSzdFNYZI+uSnlyXNVZOcKNyfahN3zyh54u5zgLA+EpnZaDNbbWbZZjaxjO33mtkKM/vEzN41s04h24rMbFnw67UwYxU5QXGx80bWVq6esoBbn13C1ryjPDimFwt+OIKvxbAg4MwJQ/jJ1WcD8N2XP+arIbWoEp26o5JTuH8puWb2YwLdUgC3AGsrOyi4hmMqcAmwCVhiZq+5+4qQ3T4C0t39sJl9HXgEuDG47Yi7J9eKJ4mawqJi/v3JFqZm5JC94yCdWzbmkS/2ZeyA9nHzCfjWIV0Y3KUlV/x+Pu+s3E7nia8nzYDx2oeuoMt9s8rdXt5iyaYN69K5ZRM6tWxM55ZNSG3ZmE4tGtO5VRPaNG0Q0RlxVVFU7Ow/UkDekQL2Hy1g/5FC8k54XhDyvJD9RwpO2L+iiR7xKtyEcRvwM+CfwefzgFvDOG4QkO3uuQBmNgMYAxxPGKFXLsAiAslIpNryC4v4x9LNTJubw4Y9hznz9Kb8fvwAruxzRo2tzahJZ7drxooHL+PsBwLjGj1//Cbv3HsR3dok9oTE6r6xHzhaSNbmPLI251XpuHopRqeWTejUojGpwWRTknTaN29U7UWT1ak1lqzCnSW1F/hmNc7fHtgY8nwTMLiC/W8HQkfKGppZJlAIPOzuM0sfYGZ3Eii9TmpqajVClGRx5FgRL36wgenzctm2/yj9OpzKpCvPYdRZp1MnDhNFqMb1A+Ma46a/z6LcPYx6fB4Pf6EP4wYl7+/0uoevLLN93+FjrNt9mPW7D7FuV/Df3YdYv/swuysodlhQ5GTvOEj2juhM4jQLrLM5tVHgq1mjuqWe16NZw7o0C3lesq1pw7o0qFu9q8hYJrBwZ0m9DVzv7vuCz5sDM9z9ssoOLaOtzOswM7sFSAeGhTSnuvsWM+sKzDazLHfPOeFk7tMJDMqTnp6eeNd4ctIOHC3guffX88yCQEHAwV1a8Oj1fRnarVXcdF+Ea8ad5/O3ReuZNHM5E/+ZxetZW/nr7RV9xko+pzWuT//G9Y9PCgjXkWNFbNhzOJhcDp2QdCqrEFCR8hJbbRRul1SrkmQBgSsOMwvnfhibgI4hzzsQWAB4guCd/O4Hhrn78SL77r4l+G+umc0BBgA5pY+vjSbNzOLFxRspcifFjPGDOzJ5bJ9YhxVVew8d49mFa3n2vXUcOFrIsB6tuWdkN87t3KLyg+PYLed1YlCXFlz6m3nMX7OLzhNfZ+WDo2lUP/HHNSKpUf0UzmzbtErlWyAw1tXtfk0BDke4nXrFZnb82jg4kymcT/NLgO5m1sXM6gPjgBNmO5nZAOBp4Bp33xHS3tzMGgQftyJwA6fQwfJaa9LMLP62aANFHvgRFLnzt0UbmDQzK8aRRceO/Uf5xesrGPKr2fx+djZD0lrx73uG8pfbBiV8sijR4/SmrPr56OPPz3rgTVZt2x/DiJJX3SQqCBlp4V5h3A8sMLO5wecXERw3qIi7F5rZPcBbQArwjLt/amYPApnu/hrwKHAK8HKw+2CDu18DnAU8bWbFBBLbw6VmV9Vazy/aUG57Ml9lbNxzmKfn5fBS5iYKi4q5pl877h7RjR6nJ/bgcHka1kth3cNX8pVnP2DO6p2M/u18HhzTK9ZhSS0W7qD3m2Y2EDgv2PQdd98V5rGzCNw/I7TtgZDHo8o57j0ged/9TkJtu+1lzs6DPDUnh5kfbcYsUHTwrmFpdGqZHKujK/PnWwfxUuZGfvDKJzzwqoorSOxUmDCCXU/73D3P3XeZ2SFgLNDDzKao3LlE0oot+5k6J5tZWVtpULcOXzq/E3de1JUzTv38PbST3Q3pHTmnU3Mufmxu5TuLREhlVxgvAdcCeWbWH3gZeAjoBzwJfDWy4Ult9NGGvUzNyOadlTs4pUFd7hqWxu1Du9DqlAaxDi2m0lqfwurJozlz0puxDkVqqcoSRqOSmUoEFtQ94+6PmVkdYFlkQ5PaxN1ZlLuHKRlrWJi9m9Ma1+M7o3rwlQs6c2pj3fqzRHXn7ovUhMoSRugk9pHAfQDuXpxo89slPrk7c1bvZEpGNkvX76V10wb86Iqe3Dy4E010u0+RuFLZX+RsM3sJ2Ao0B2YDmNkZgMYv5KTMytrK1IxsPt2yn/anNeLnY3pxfXrHpKijJBIpzRqksD+/qMz2SKssYXybQCHAM4Ch7l5yz8y2BKbailTb3c9/SJdWTXjkur5cO6B9tWv91DaxfMOQ2CvrZ19Re02qMGG4uwMzymj/KGIRSa3xxPgBXBGnBQHjWSzfMKR2UyexxMzV/drFOgQRqQIlDImI/UcL+Ov762MdhkjSMcpepBuN6/Rwq9VeC8wKLQwoUpY9wYKAfw4WBBSRmnXzean8rYzyQDefF/lS+OGOMl4DfGZmfzWzK81MVyZxyj02BUK27z/K5P+sYMjDs5mSkc3Qbq34zzeGxiQWkWQ2eWwfbjkvlZTg0oYUM245LzUqdeTCrSV1q5nVAy4HbgKeNLO33V0rveNMRbfEjKY3lm/jjeXbYh2GSFKaPLZPTAqNhn2l4O4FZvYGge6zRgRutaqEISJSS4Q7hjGawL0sRgBzgD8CN0QuLKmuSN8d7NMteTyZkcOs5YGCgDcNChQEbHtqwzL31/2QRZJHuFcYXyGwHuNrGviunT7csJeps7N5d1WgIODXh6VxmwoCitQq4Y5hjKtou5m97+7n10xIEi/cnfdzdjMlI5v3cgIFAe+9pAdfPl8FAWMpxez43RZLt4tEUk3Ndiq7P0ISkruTsXoHT8zO5qMN+2jdtAH3X3EWNw1OVUHAODB+cMcyp1WOH9wxBtEkh+5tmrBmx6Ey2+V/auqvP1lv9larFBU7by7fxtSMbFZsDRYEHNub68/poIKAcaRkdsyLizdS5E6KGeMHd0zq2/NG2tv3DueSx+eckDS6t2nC2/cOj11QcUgfF4WComJeW7aFJ+dkk7PzEF1bNeHR6/oytgYKAsZyVWpVJEqcJWI1rTKZKTlUrqYSRrz+XUkFjhYU8crSTUybm8OmvUfo2bYpU24awOW9a64gYKLcfzyWq2eTmcZbkktl9/TuBpzu7gtLtV8IbHH3nGDTlyIUn0TA4WOFvLB4A9Pn5bLjQD79O57Gz67pxciebajpG2Ml2id3qVkab0kulV1h/Bb4URntR4LbrgZw9+U1HJdEQN6RAv76/jqeWbiOPYeOcX7Xlvzmxv5ckNayxhNFiUS5wni+jDe1knZ1/VSfxluSS2UJo7O7f1K60d0zzaxzRCKSGrf7YD7PLFzLc++t50B+ISPObM09I7txTqcWsQ4tbiRKYktEGm9JHpUljIqmyzaqyUCkcu5Ozs7PT/0rz/b9R5k+L5cXFm/gaGERl/duy93Du9G7/akUjVpKAAAPn0lEQVQRjPJE6sMWSR6VJYwlZnaHu/8htNHMbgeWRi6s2m3LviPMXrWDjFU7yFi9g+IqfszduOcwT83N4ZXMTRS5M6ZfO+4ekUa3Nk0jE3AF1IctkjzCuaf3v8zsZv6XINKB+sC1kQwsVqI1F3v3wXzmfraTjNU7yVi1g4P54d07ov1pjdi870i52+99aRmvLttCihnXpXfgrovSSG3ZuKbCrrJE6cO+pZxZUrdolpTIcRbO/RPMbATQO/j0U3efHdGoqik9Pd0zMzOrfXzpZFGiuknjYH4hC9bsImPVDmav3sHOA+GV4WrRpD7Dz2zNyJ5tuLB7a05tdGIZjooK+jWsV4ebB3fijgvLLwgon1fTP3uRRGJmS909vbL9wq0llQFknHRUca6sN4yK2iGwlmHx2j3Hu4/W7z4c1ms1rFeHkT3bMPzMNgzv0Zo2zWrmzX3hD0fSUgUBq6w6P3uR2kYrvcOUuW4Ps1ftYPaqHazadiDs44b1CFwpjDizTY10DVV2RahkISKRooQRpuumvV/utkFdWjDizDaM7NmGHqefEpE1De7O7FWBgoAiIrGghBGm3u2bMfLMNozo2Ya+HU6rsdIZlSkqdt5YvpWpGTmsDBYEFBGJBSWMEBWtGfjPNy6MaiwFRcXM/GgzT83NIXfnIbq2bsKvr+/HmP7t6H7/G1GNpTbQehGRyilhhIiHNQNHC4p4eekmps3JYfO+I5x1RjOm3jSQ0b3bRu2qpjaKh5+9SLxTwggxeWwflm/OY9nGPICorhk4lB8oCPiH+YGCgANST+PnY3sx4syaLwgon5co60VEYiniCcPMRgO/A1KAP7r7w6W23wt8FSgEdgK3ufv64LYvA5OCu052979EOt5Le7Vl2cY87hqWxsTLe0b65cg7UsBz763jmYVr2Xu4gAvSWvLbG/tzfgQLAkrZVPNIpGIRTRhmlgJMBS4BNhEoNfKau68I2e0jIN3dD5vZ14FHgBvNrAXwEwIryx1YGjx2byRjjpbdB/P504K1/PX9QEHAkT3bMGFEN87p1DzWoYmIlCnSVxiDgGx3zwUwsxnAGOB4wgguCiyxCLgl+Pgy4G133xM89m1gNPBihGOOqG15wYKAH6wnv7CYK3qfwd0j0ujVLnoFAUVEqiPSCaM9sDHk+SZgcAX73w6UTAEq69j2pQ8wszuBOwFSU+O37s+G3YGCgP9YGiwI2L8ddw/vRrc2p8Q6NBGRsEQ6YZTVCV/mUmUzu4VA99Owqhzr7tOB6RCoJVW9MCNnzfYDPDknh9c+DhQEvD69A3cNS6Nji9gVBBQRqY5IJ4xNQOi8xA7AltI7mdko4H5gmLvnhxw7vNSxcyISZQQs35zH1Ixs3vx0Gw3rpnDrBZ2546KunF5DNaNERKIt0gljCdDdzLoAm4FxwE2hO5jZAOBpYLS77wjZ9BbwSzMrGQW+FLgvwvGetMx1e5iSkc2c1Ttp2qAuE4Z347ahXWjRpH6sQxMROSkRTRjuXmhm9xB4808BnnH3T83sQSDT3V8DHgVOAV4OTiPd4O7XuPseM/s5gaQD8GDJAHi8cXcWZu9mSsYaFuXuoUWT+nz/sjP50vmdaNawXuUnEBFJABFfh+Hus4BZpdoeCHk8qoJjnwGeiVx0J8fdeXflDp7IyObjjfs4vVkDJl15FjcNTqVxfa2JFJHkone1aigqdmZlbWVqRjarth2gQ/NG/OLa3lx3Tgca1E2JdXgiIhGhhFEFBUXF/OujzUybk0PurkOktW7CY9f345r+7aiXUifW4YmIRJQSRhiOFhTxcuZGps3NZfO+I5x9RjOevHkgl/VSQUARqT2UMEr50/xcAKbNzWHa3BxanVIfM2PngXwGpp7G5LG9GX5m65jWeVIpbhGJBSWMEJc8PofdhwpOaNt18BiN6tXhhTsGc37X+CgIqFLcIhILShgh1uw4VGb7kYJiLkhrFeVoyrc4d3eV2kVEaoJGahNQeYmtvHYRkZqghCEiImFRwhARkbAoYYiISFiUMBJQ9zZNqtQuIlITlDASUO7Ow1VqFxGpCUoYCaisRXsVtYuI1AQljARU3opurfQWkUhSwkhA5a3o1kpvEYkkJQwREQmLEkYCenHxxiq1i4jUBCWMBKRBbxGJBSWMBKRBbxGJBSWMEOW93cbb27AGvUUkFlTePER5HTrx1tEzeWwfIDBmUeROihnjB3c83i4iEglKGAlq8tg+ShAiElXqkhIRkbAoYYiISFiUMEREJCxKGCIiEhYlDBERCYsShoiIhEUJQ0REwqKEISIiYVHCEBGRsChhiIhIWJQwREQkLBFPGGY22sxWm1m2mU0sY/tFZvahmRWa2XWlthWZ2bLg12uRjlVERMoX0eKDZpYCTAUuATYBS8zsNXdfEbLbBuArwPfKOMURd+8fyRhFRCQ8ka5WOwjIdvdcADObAYwBjicMd18X3FYc4VhEROQkRLpLqj0QeqPpTcG2cDU0s0wzW2RmY2s2NBERqYpIX2GUdbO6qtyPKNXdt5hZV2C2mWW5e84JL2B2J3AnQGpqavUjFRGRCkX6CmMTEHrf0A7AlnAPdvctwX9zgTnAgDL2me7u6e6e3rp165OLVkREyhXphLEE6G5mXcysPjAOCGu2k5k1N7MGwcetgCGEjH2IiEh0RTRhuHshcA/wFrASeMndPzWzB83sGgAzO9fMNgHXA0+b2afBw88CMs3sYyADeLjU7CoREYmiiN/T291nAbNKtT0Q8ngJga6q0se9B+im1SIicUIrvUVEJCxKGCIiEhYlDBERCYsShoiIhEUJQ0REwqKEISIiYVHCEBGRsChhiIhIWJQwREQkLEoYIiISFiUMEREJixKGiIiERQlDRETCooQRIsXKukFg+e0iIrWJEkaI8YM7VqldRKQ2ifj9MBLJ5LGB22+8uHgjRe6kmDF+cMfj7SIitZm5e6xjqDHp6ememZkZ6zBERBKKmS119/TK9lOXlIiIhEUJQ0REwqKEISIiYVHCEBGRsChhiIhIWJQwREQkLEoYIiISFiUMEREJS1It3DOzncD6GjrdOcDSGjpXJCVKnJA4sSZKnKBYIyFR4oSai7WTu7eubKekShg1yczc3eO+6mCixAmJE2uixAmKNRISJU6IfqzqkhIRkbAoYYiISFiUMMqXKH11iRInJE6siRInKNZISJQ4IcqxagxDRETCoisMEREJixJGCDM7amYe8lUc65jKY2brS8Ua15eKZvZygsSZSN/TvFLxFsY6ptLM7Fjp72mc/10Vl4q1pqbp16hScUbt+6mEcaJioDA4TW0PYGZ2LMYxlaczsCsY64cA8fiGEeKLsQ6gCg65uyXA1MpmQMm0ypXAoRjH8znuXj/ke/lKsDku/6bM7AhgwOaQn33c3Z/ZzAoIxLkNmEcU36c0hlGB4CfMYndPiXUsFTGzrUBboMjd4+62u8E/xIYEEnKdeH4jDv7MD7n7KbGOpSJmtgdoTuD7mRB/xGZWRBz//EN+Tw8AHwEXEYd//6W/jyVXGO4e8QsAXWGUw8zygw/zYhpIBUq6eQgkC+IxWQQ1BOL56qe0JvHedQI0Df4b2jWxPaYRVa4OcTwDyd0bEYivKYFkQbwli6AiADM7EPywaMGviFPCKEOwxEh9Ap8uWsQ6nvK4+/XBTxn74filalwJfhrC3evFOpYwHQ1+Tw8TuNSP10RX8gZRFPKJvU2sgqmMma0IPjwS00AqYGZ5BL6vh/hfN2/cfWhw9/oEEtspBD8sRou6pEoxs6XAQAJ9wwmTUKN5WVoV5Q0cx2u3RKhg7HH5e2Bmh4FGId0ShUBKvH5f4707Csrt6on7saxo/p7G3R9CLJmZEUgWcffGW5qZHQx+IsLMNhH4ZBR32b9kwDP4R1eS1OLyD9DMtpvZ7pLHwea4+54CuHtjCMxCCjbFY9dJqLjujgpyCIwJBt8L4vJvCsDMDgX/PRpsOhCN143XPu9YOd6lE/LJOO4GvYIaAHVD4vQ4jTORnAbUT6DvaSFQLyTe/bEMpjzBfnaI4/FACIwBhowJlnRF5cYwpIo0LvUedWo0XlRdUiIiEpa47nYREZH4oYQhIiJhUcIQEZGwKGGIiEhYlDBERCQsShiS0IIlMR4Lef49M/tpDZ37z2Z2XU2cq5LXud7MVppZRkhbHzNbFvzaY2Zrg4/fqeZrvGVmTSvfU6R8ShiS6PKBL5hZq1gHEsrMqrJ+43bgbncfUdLg7lnu3t/d+wOvAd8PPh9VnXjc/TJ3j8riLkleShiS6AqB6cB3Sm8ofYVgZgeD/w43s7lm9pKZfWZmD5vZzWb2gZllmVlayGlGmdn84H5XBY9PMbNHzWyJmX1iZl8LOW+Gmb0AZJURz/jg+Zeb2a+CbQ8AQ4FpZvZoOP9hM6tjZo8Hz5NV8n80s1HB159pZivMbGpwxTJmtsnMTgs+vjUY98dm9mywbVzwfB+HXumIhNJKb0kGU4FPzOyRKhzTDziLwH1PcoE/uvsgM/sW8A3g28H9OgPDgDQgw8y6Af8H5Ln7uWbWAFhoZv8N7j8I6O3ua0NfzMzaAb8CzgH2Av81s7Hu/qCZjQS+5+6ZYcZ+PXB28P/QGlhiZvOC2wYHt20E3gbGADND4ugH/BC4wN33mFlJcc2fAMPdfXtJYhEpTVcYkvDcfT/wHPDNKhy2xN23uns+kAOUvOFnEUgSJV5y92J3X0MgsfQELgX+z8yWAYuBlkD34P4flE4WQecCc9x9p7sXAs8TLKFdDUOBF9y9yN23AQuA9OC2Re6+zt2LgBnBfUONBP7u7nsASv4FFgLPmdlX0fuClENXGJIsfkugJPWzIW2FBN/8gl0z9UO25Yc8Lg55XsyJfxela+c4gaJ033D3t0I3mNlwyr/rXU0WXKzoXGXFW/rYsuoB3UHg6uQq4GMz6+vue6sfoiQjfZKQpBD8pPwSgQHkEusIdAFBoGumOvfkuD44ZpAGdAVWA28BXzezegBm1sPMmlRynsXAMDNrFRwQHw/MrUY8ELgt57jgWMrpwBCgpDvrPDNLDb7GDQSuPkK9Ezy2RTD2ki6pru6+CPgxgS6z9tWMTZKYrjAkmTwG3BPy/A/Aq2b2AfAu1bvn9WoCb+ynA3e5+1Ez+yOBbqsPg1cuO4GxFZ3E3bea2X1ABoFP+bPc/dVqxAOBe2OfB3xM4GrhXnffERzffo/A96EXMIfADKvQOErGeuZZ4B4aSwkk2d+YWZdgbP919+XVjE2SmKrViiQJMxsF3OPuFSYvkepSl5SIiIRFVxgiIhIWXWGIiEhYlDBERCQsShgiIhIWJQwREQmLEoaIiIRFCUNERMLy/3Vlcf5LRmtwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# filter the results\n",
    "results_for_vis = pd.read_csv(\"lda_tuning_results_overall.csv\")\n",
    "#results_for_vis = results[results[\"Alpha\"]!='symmetric']\n",
    "#results_for_vis = results_for_vis[results_for_vis[\"Alpha\"]!='asymmetric']\n",
    "#results_for_vis = results_for_vis[results_for_vis[\"Beta\"]!='symmetric']\n",
    "#results_for_vis = results_for_vis[results_for_vis[\"Beta\"]!='asymmetric']\n",
    "\n",
    "# now the plot\n",
    "import matplotlib.pyplot as plt\n",
    "plt.plot(np.array(results_for_vis[\"Topics\"]), np.array(results_for_vis[\"Coherence\"]))\n",
    "plt.scatter(np.array(results_for_vis[\"Topics\"]), np.array(results_for_vis[\"Coherence\"]))\n",
    "plt.ylabel(\"C_v Score\")\n",
    "plt.xlabel(\"Number of Topics\")\n",
    "plt.xticks(np.array(results_for_vis[\"Topics\"]))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHWxJREFUeJzt3X+QHOV95/H3hxUiWJgqbCTHQQjxQ64zMdjYgyCx+REOBdmuEiJnEuQoZ3zE+C7HXeV0ThXEsjkIdaSgQt1djCuAy5XC5KwAxjrlLMAqgpzgsyitIhtF4rCEDEKRbQkwwYBPtlbf+6N72NbszG7varr70c7nVaXSzjPdM9+d7ZnvPN/n6acVEZiZmU3kqKYDMDOzI4MThpmZleKEYWZmpThhmJlZKU4YZmZWihOGmZmV4oRhZmalOGGYmVkpThhmZlbKjKYD6KcTTzwx5s+f33QYZmZHlE2bNr0YEbMn2m5aJYz58+czPDzcdBhmZkcUSc+X2c4lKTMzK6XyhCFpsaRnJO2QdH2X+1dI2ibpKUmPSTqlcN88Sd+U9HS+zfyq4zUzs+4qTRiShoA7gQ8DZwLLJJ3ZsdlmoBURZwMPArcV7rsXuD0i3g0sBPZWGa+ZmfVWdQ9jIbAjInZGxM+BVcDlxQ0i4vGIeCO/uQGYC5AnlhkRsS7f7rXCdmZmVrOqE8ZJwAuF27vztl6uAR7Of34X8IqkhyRtlnR73mM5hKRrJQ1LGt63b1/fAjczs0NVnTDUpa3rFZskLQdawO150wzgAuAzwLnAacDVYx4s4u6IaEVEa/bsCWeFmZnZFFWdMHYDJxduzwX2dG4k6VLgs8CSiNhf2HdzXs46AKwG3l9xvGZm1kPVCWMjsEDSqZJmAlcBa4obSDoHuIssWezt2PcESe1uwyXAtorjNTOzHipNGHnP4DrgUeBp4P6I2CrpZklL8s1uB44DHpD0XUlr8n1HyMpRj0naQlbeuqfKeM3MrDdFdB1SOCK1Wq3wmd5mZpMjaVNEtCbazmd6m5lZKU4YZmZWihOGmZmV4oRhZmalOGGYmVkpThhmZlaKE4aZmZXihGFmZqU4YZiZWSlOGGZmVooThpmZleKEYWZmpThhmJlZKU4YZmZWihOGmZmV4oRhZmalOGGYmVkpThhmZlaKE4aZmZXihGFmZqU4YZiZWSlOGGZmVooThpmZlTKj6QDMzGxyVq7ewleffIGRCIYklp13MrcsPavy53XCMDM7gqxcvYX7Nux68/ZIxJu3q04aLkmZmR1BvvrkC5Nq7yf3MMxsQk2VQGyskYhJtfeTexhmNq52CaT9gdQugaxcvaXhyAbTkDSp9n5ywjCzcTVZArGxlp138qTa+8kJw8zG1WQJxMa6ZelZLJgz65C2BXNmeZaUmTVvSOqaHOoogaSo6fGclau3sH3v64e0bd/7OitXbznyZ0lJWizpGUk7JF3f5f4VkrZJekrSY5JO6bj/eEn/JOkLVcdqZmM1WQJJTQrjOU2WCCtNGJKGgDuBDwNnAsskndmx2WagFRFnAw8Ct3Xc/yfAt6qM08x6u2XpWSw/f96bPYohieXnzxvIWVIpjOc0WSKsuiS1ENgRETsBJK0CLge2tTeIiMcL228AlrdvSPoA8A7gEaBVcaxm1sMtS88ayATRKYXxnCZLhFWXpE4Ciql3d97WyzXAwwCSjgL+DPijyqIzM5uEJqe0tk3nWVLdXsWuqVjScrJexO150x8AayNi3L6epGslDUsa3rdv32EFa2Y2nhTGc5osESoq7EpJ+jXgv0TEZfntGwAi4taO7S4F/hy4KCL25m1/BVwAHASOA2YCX4yIMQPnba1WK4aHh6v4VczMgOZnSVVB0qaImLDsX3XCmAF8H/iXwD8BG4GPR8TWwjbnkA12L46I7T0e52qygfHrxns+JwwzGwT9TlplE0alg94RcUDSdcCjwBDw5YjYKulmYDgi1pCVoI4DHlDWxdoVEUuqjMvMbKqa7mE0uVptpT2MurmHYWZV6vywbqtzmvHpN6ztOUvq2Vs/MqXHLNvD8NIgZmYlDfp5GE4YZmYlpXIexmTa+8kJw8ysJJ+HYWZmpfg8DA96m5mV1vQsqSokcR5G3ZwwzMwmL4nzMMzs8KXwjfbsGx/h1f0jb94+/pghnrppca0xpBLHojvWH3I9igVzZrFuxcW1xtAUj2GYJSyF6y90fkgDvLp/hLNvfKS2GFKJozNZQHbxokV3rK8thiY5YZglLIV5/50f0hO1T+c4OpPFRO3TjROGWcJSmPdv1uaEYZawFOb9m7V50NssYcvOO7nr2kV1zvs//pihrmWf448Zqi2GVCyYM6tr+WnBnFm1xtHURAj3MMwSlsL1tJ+6afGY5NDULKmmnXfa2yfVXoUmJ0K4h2FmE0ohOTR5Leu28SYh1JXEu/U42+1Vx+AehlnCUphWm4oUluUY9EkIThhmCUthWm0qUijPDfokBJekzBI26N9oO92y9KxG121KYRJCk9zDMEvYoH+jtbF6zU6rY9aaE4ZZwlKo26dk5eotnH7DWuZf/w1Ov2Ft7WM5KZQIX//5wUm195NLUmYJa5dfml58MAWd19NuTwAAans9UigRNhmDE4ZZ4pqu26cihSmtKUztbTIGl6TMbEJNl4IgjW/3KZQIm4zBPQwzG1cKpSBI49t9CiXCJmPwFffMbFyn37C25wf1s7d+pLY4OhNXW93nYkxHvuKe2TTR9BX3UigFQRrf7gedE4bZOJr+sE6lHJQKTwBolge9zXpIYR2nFOb9m7U5YZj1kMKHdQrlIJ9tbm0uSZn1kMKHdQpSWj+p6RLhoHPCMOshhWmcArqlpzq/26cy2OzxnOY5YZj1cNrst3S9HOdps99SWwy9+jJ193FSGGxO4UzvQVf5GIakxZKekbRD0vVd7l8haZukpyQ9JumUvP19kr4jaWt+3+9UHatZ0c59b0yqvQoePxjlEmHzKu1hSBoC7gQWAbuBjZLWRMS2wmabgVZEvCHp3wG3Ab8DvAH864jYLulXgE2SHo2IV6qM2dLRdL06hQ+olMYPmpZCiTAVTb03qu5hLAR2RMTOiPg5sAq4vLhBRDweEe2vbBuAuXn79yNie/7zHmAvMLvieC0RKUxpTeHbfQpXmUtFCus4paDJ90bphCHpFEmX5j8fK+mtJXY7CSgWHnfnbb1cAzzc5bkXAjOBZ8vGa0e2FKa0pvIBdcvSs3j21o/w3J9+lGdv/chAJgtw8mxr8r1RKmFI+hTwIHBX3jQXWF1m1y5tXfvzkpYDLeD2jvZ3Al8BPhkRY64QIulaScOShvft21ciJDsSpFAOMktRk++Nsj2Mfw98EHgVIC8VzSmx326g+HVsLrCnc6O85/JZYElE7C+0Hw98A1gZERu6PUFE3B0RrYhozZ7titV0kUI5KIVejo1KoUyZgibfG2UTxv58DAIASTMoN7NvI7BA0qmSZgJXAWuKG0g6h6znsiQi9hbaZwJfB+6NiAdKxmnTRArlIPdy0uIEnjkSrofxLUl/DBwraRHwB8DfTLRTRByQdB3wKDAEfDkitkq6GRiOiDVkJajjgAeUZchdEbEE+G3gQuDtkq7OH/LqiPhu+V9v8pqemWOZVE4WS4GPyUwqCbzpv8ctS8/iyZ0vHXKO0II5s2qJoWzCuJ5sQHoL8GlgLfClMjtGxNp8+2Lb5ws/X9pjv/uA+0rG1xc+kzQtKZws1jQfk2lJ4e+xcvWWMSeUbt/7OitXb6k8hglLUvm5FPdGxD0RcWVEfCz/edr1y1Pp8qZwOUxLQyrHZAp6VejrPAsjhb9H0rOkImIEmJ2PKUxrKXR5PbBnRSkck6lIYZmUFP4eTcZQtiT1HPBtSWuAN/tCEXFHFUE1JYWF3v6qy1m97XaXIOp1/DFDvLp/pGt7XVI5u3nRHevH1MzXrbi41hhSeC0GPYays6T2AP873/6thX/TSgrfYFKIwTLdksV47VXotdBhnQsgdiYLyGrmi+5YX1sMkMbMuUGPoVQPIyJuAsjP7o6IeK3SqMwMSGMBxG4r9o7XXpUUZs4NegylEoak95Cdbf22/PaLZAsDbq0wttql0N00K0qhZp6SFGbODXIMZUtSdwMrIuKUiDgF+M/APdWF1YwUuv8pzASxzII5sybVXgUfD5aSsoPesyLi8faNiFgvqb53TU1S6P6fMWdW167+GTV+SEHzJyelYN2Kixsf7PWYlqWkbMLYKelzZGUpgOXAD6oJqTkpdP9TSFopnJyUirpnAqXIpVprK5sw/g1wE/BQfvvvgE9WElGDUnhjpJC0fCnMdKRwTKZ0EacUer4pxNCUsrOkfgL8x4pjaVwKb4wUPiBSSFqWSeG64inMDII0er4pxNCksrOk1gFXti+PKukEYFVEXFZlcIPISWvUIH+Ta0uhRAlpzAxKoeebQgxNKjtL6sTitbTzHkeZ62EcUVJYJyaFq4qlcHKSl0jJuLc3KoXXIoUYmlR2DOOgpHkRsQuyy7UyDSdqpHIwNP1tLoUSxKB/k2tLpbeXghReixSWD2pS2YTxWeAJSd/Kb18IXFtNSM1J4YAEl2IgneTdtBRKlKlI4bUY9GnOpUpSEfEI8H7gr/N/H4iIR6sMrAkuxaQTQwqXaE1BCiXKVPi1aN64PYy89PRKRPxzRLwo6XVgKfAuSV8oXrZ1OnApJpPCirkpzA6y9DRdrk2lCtGUiUpS9wNXAP8s6X3AA8CtwHuBLwK/X2149Wv6gEyhFJNCtzuV2UFNG/RpnKlJoSzWpIlKUsdGxJ785+Vk1+T+M7KT9hZWGtmA8tpBmRQSZwpSmLlnowa9LDZRD6P4OXUJcANARBzUgHTB6pbCt/sUDHrXv82JMz1NVyGaNFEP428l3S/pvwMnAH8LIOmdwLQav7BRy8+fN6n2KqQwASEFHvy3lEyUMP6QbP2o54APRcQv8vZfJptqa32WwgdECt3uFGJIgROnpWTcklREBLCqS/vmyiIacKkMqj2586VDptU+ufOlWp8fBrvr35bCzD2zNsU0qoW2Wq0YHh5uOozD1vSJe92u4Qz1XwvCzOohaVNEtCbaruyZ3lajpr9Zp3INZzNLS9nVaq8A1kbE/orjMUtK0709s5SUXa12CfB9SV+R9FFJ7pnYtJfCEilmKSm7ltQngTPIzvT+OPCspC9VGZg1Z0GP64f3ap+ufNKc2aFK9xQi4heSHiY7h+xY4HKm4dIgll3HunPgu4kB76bLQT5pLj1NHxODHkPZMYzFwFXAbwDrgS8Bv11dWNa0pmdDpbCGUipnm6fwAZWCFI6JQY+h7BjG1cBq4F0R8YmIWBsRB6oLywZdCuWgWTO7vz16tVfB4yijUjgmBj2GsmMYV0XE6l6zpCR9p9e+khZLekbSDknXd7l/haRtkp6S9Fi+pHr7vk9I2p7/+0SZWG16SKEc9Or+kUm1VyGFD6hUpHBMDHoM/fqq9EvdGiUNAXcCHwbOBJZJOrNjs81AKyLOBh4Ebsv3fRtwI3Ae2cq4N0o6oU/xWuJSWCIlBSl8QKUihWNi0GPoV8LodfQuBHZExM78YkuryAbLR3eMeDwi2hc52ADMzX++DFgXES9HxE+AdcDiPsVrifMaSpkUPqBSkcIxMegxVF2MPQko9p135229XAM8PMV9bRpJYfHBFKYXp/ABlYoUjolBj6Eva0lJ2hwR53RpvxK4LCJ+P7/9e8DCiPgPXbZdDlwHXBQR+yX9EXBMRNyS3/854I38Ak7F/a4FrgWYN2/eB55//vnD/n3M2jy92AZBX9aSknQG8I6I+HZH+wXAnoh4Nm/6vR4PsRsofhWaC+zp3EjSpWTLpV9UGFjfDVzcse/6zn0j4m7gbsgWHxzv9zlSpPABkcIHZQpS+J2bXlvMrG2iktR/A37apf1n+X0ARMQ/9th/I7BA0qmSZpKdy7GmuIGkc4C7gCURsbdw16PAb0o6IR/s/s28bVpLYRplt9Vqt+99nUV3rK8tBjNLz0Qn7s2PiKc6GyNiWNL8iR48Ig5Iuo7sg36I7JrgWyXdDAxHxBrgduA44IH8sq+7ImJJRLws6U/Ikg7AzRHxctlf7Eg13jTKur5lerVa65RCr9eaN1HC6DpdNndsmSeIiLXA2o62zxd+vnScfb8MfLnM80wXnkZpqUnh7GZLw0QJY6OkT0XEPcVGSdcAm6oLqzlN1+5TWY7CrC2FXq+lYaKE8YfA1yX9LqMJogXMBK6oMrAmjFe7rytppHCJ1gVzZvW84l6dzr7xkUPOqj7+mCGeusmn4tTNvV5rG3fQOyJ+HBG/DtwEPJf/uykifi0iflR9ePVKoXafwjzvdSsuHpMc6u5pdSYLyJbkOPvGR2qLwTI+edDaSq1WGxGPA49XHIvlUphG2fR00hTWcbJMCr1eS4OvnGdm42p/efEsKXPCKEildm+WmhR6vdY8J4yCdSsuZv713+jaXicP9ppZiuq7EswRoNeAap0DrR7szXig1Sw9ThgFKQy0phBDCrxKq1l6XJKyJHmg1Sw97mGYmVkpThgFxx8zNKn26RpDClJYtdfMDuWEUfDUTYvHfDDXPUMphRhSMN76RWbWDI9hdEjhgzmFGJrm9YvM0uMehiXJ02rN0uOEYUnytFqz9LgkZUnytFqz9CimUU241WrF8PBw02GYmR1RJG2KiNZE27kkZWZmpThhmJlZKU4YZmZWihOGmZmV4oRhZmaleFqtdbXojvWHXH1wwZxZjV/n28ya5R6GjdGZLAC2732dRXesbyYgM0uCE4aN0e265uO1m9lgcMIwM7NSnDDMzKwUJwwzMyvFCcPG8NLiZtaNE4aN4aXFzaybyhOGpMWSnpG0Q9L1Xe6/UNI/SDog6WMd990maaukpyX9D8lfcetwy9KzWH7+vDd7FEMSy8+f56XFzQZcpSfuSRoC7gQWAbuBjZLWRMS2wma7gKuBz3Ts++vAB4Gz86YngIuA9VXGbJlblp7lBGFmh6j6TO+FwI6I2AkgaRVwOfBmwoiI5/L7DnbsG8AvATMBAUcDP644XjMz66HqhHES8ELh9m7gvDI7RsR3JD0O/JAsYXwhIp7uf4iHWrl6i6/yZmbWRdVjGN3GHEpd4k/SGcC7gblkiecSSRd22e5aScOShvft23dYwa5cvYX7NuxiJL8K4UgE923YxcrVWw7rcc3MpoOqE8ZuoDi1Zi6wp+S+VwAbIuK1iHgNeBg4v3OjiLg7IloR0Zo9e/ZhBfvVJ1+YVLuZ2SCpOmFsBBZIOlXSTOAqYE3JfXcBF0maIelosgHvSktSIz2ub96r3cxskFSaMCLiAHAd8CjZh/39EbFV0s2SlgBIOlfSbuBK4C5JW/PdHwSeBbYA3wO+FxF/U2W8PmHNzKy3yq+HERFrgbUdbZ8v/LyRrFTVud8I8Omq4ytadt7J3LdhV9d2M7NB5wsoFbRnQ3mWlJnZWIppVJ9vtVoxPDzcdBhmZkcUSZsiojXRdl5LyszMSnHCMDOzUpwwzMysFA96W7K8TItZWpwwLEntZVra2su0AE4aZg1xScqS5GVazNLjhGFJ8jItZulxwrAkeZkWs/Q4YViSfF1xs/R40NuS5GVazNLjpUHMzAaclwYxM7O+csIwM7NSnDDMzKwUJwwzMyvFCcPMzEpxwjAzs1KcMMzMrBQnDDMzK8UJw8zMSnHCMDOzUpwwzMysFCcMMzMrxQnDzMxKccIwM7NSnDDMzKwUJwwzMyvFCcPMzEpxwjAzs1KcMMzMrJTKE4akxZKekbRD0vVd7r9Q0j9IOiDpYx33zZP0TUlPS9omaX7V8ZqZWXeVJgxJQ8CdwIeBM4Flks7s2GwXcDXwP7s8xL3A7RHxbmAhsLe6aM3MbDwzKn78hcCOiNgJIGkVcDmwrb1BRDyX33ewuGOeWGZExLp8u9cqjtXMzMZRdUnqJOCFwu3deVsZ7wJekfSQpM2Sbs97LIeQdK2kYUnD+/bt60PIZmbWTdUJQ13aouS+M4ALgM8A5wKnkZWuDn2wiLsjohURrdmzZ081TjMzm0DVCWM3cHLh9lxgzyT23RwROyPiALAaeH+f4zMzs5KqThgbgQWSTpU0E7gKWDOJfU+Q1O42XEJh7MPMzOpVacLIewbXAY8CTwP3R8RWSTdLWgIg6VxJu4Ergbskbc33HSErRz0maQtZeeueKuM1M7PeFFF2SCF9rVYrhoeHmw7DzOyIImlTRLQm2s5nepuZWSlOGGZmVooThpmZleKEYWZmpThhmJlZKdNqlpSkfcDzfXq4DwCb+vRYjsExOIb+SiGO6RTDKREx4VIZ0yph9JOkiIhuS5s4BsfgGBqWQhyDGINLUmZmVooThpmZleKE0VsKtTrHkHEMGccwKoU4Bi4Gj2GYmVkp7mGYmVkpA5cwJI1IivzfwS73f6pwf+RTdZH0RLG94hgOFO4f6bivrhgOdrwOP+53DFOM4428vba/R5ftflF3DF1eh4OF+2p7HTrjqDuGjtegqRh+2BHDSN5e5/Hw444YdhXu69v7s9NAJQxJ7yb7nV8GPp41ZR9ABXcB5FPVAjgxb98IjHCYNcOSMfwE2N/jIQ7m/6qOIYD/Wngd5vQzhknE8dcRoTyOg8CxeXudfw8k/SVj3y+1xgDZcZn/K8ZSyzEh6VWyywx8M/97bKk7hsLv3z4ui+p6b/xyFkoI+Bmjx0Wdx8OcQgwHOfRCdX15f3aNbZDGMCQdAIba85bbmbv45suz8khEzJB0AfB3wFGRv1D5Pprq3OcyMXTEcjAihjraR/KYKo8hv//nwNHF5zvcGKYYx5jXvq6/R/tvQfZGPhARR9cZw0TPUccxkb8GMc7fp+7jcsz7o+7XoXP7wj5VHw8BvBAR8ySJ7HXo6/uzm4HqYdD9GuPd2vYARMTf57c3NBBDlSYbw9FUMxujVBztMkh+30/rjiFPmHQm7jpjKMTSLjeUvdRxv2NQZymmgRjQaIn0PQ3EsIP8dQCGgF80EANkl7wGOEAW0BN9jmOMQUsYU9XvA+KIUaiffrqpGCKi/U0pgLfW+dzKLhF8NPBinc/bxf8Bri28Du9sMJaPAi8BR0n6RkMxzAaIiKcbeO7TGS0HjQBHa/RS0nX5AaNJq/05vrfqJx20hNHtW3K3tl8ByEtSABd02abqGKpUKob8G6SALRFRxeVxJ/tavKOBGO7K/z+xMIg4Q/nAd00xEBEfKvwNfrWPz106htFQYm1EtMf2LmsgBsiOy373cMrGIOBHABExI297rM4YIuK0wljOX+Rtv9XHGLoatITxGwCSXpS0jOwP3zm43O5mAnwLsndIzTFUbcIY8jLMUcBPI+LsBuN4tXDzR3XHEBG/1THQCh1jGFXHkN+3vXDzH/v43KVjIBvDadfVn8zb+lmuLfXekPRa/mM//waTioFs4BtJr+e331tnDJKeLPRq/m0fn3tcAzXoDaODQfnN9sDVm4Nnkv4TcEdhl5cj4u35vp0v1s8i4i0VxHCA0aTV3qj9Rh0zM2QqtfUSMYw5MPodwxTj+H8RcWyPOCr5e3RsGxQSRo3HROfzvBERs3rEUNUx0Z6RQ3GbOmMoPlfngG6Nr8MbjM7WO+R5ajweivdDVq68p0cMU35/jolr0BKGmZlNzaCVpMzMbIqcMMzMrBQnDDMzK8UJw8zMSnHCMDOzUpwwzKZI0hX5Ehn/Ir89X9K450iU2cYsVU4YZlO3DHgCuKrpQMzq4IRhNgWSjgM+CFxDl4Qh6WpJ/0vSI5KekXRj4e4hSfdI2irpm5LaJyJ+StJGSd+T9DVJkz7hy6xKThhmU7MUeCQivg+8LOn9XbZZCPwu8D7gSkmtvH0BcGdE/CrwCvCv8vaHIuLciHgv8DRZMjJLhhOG2dQsA1blP6/Kb3daFxEvRcTPgIeAD+XtP4iI7+Y/bwLm5z+/R9LfS9pClmiqWGTQbMpmTLyJmRVJejtwCdkHfHuxygC+2LFp57o77dvFheRGGF2X6C+BpRHxPUlXAxf3L2qzw+cehtnkfQy4NyJOiYj5EXEy2fUJ5nZst0jS2/IxiqXAtyd43LcCP5R0NFkPwywpThhmk7cM+HpH29eAP+5oewL4CvBd4GsRMTzB434OeBJYB/zfPsRp1lderdasAnlJqRUR1zUdi1m/uIdhZmaluIdhZmaluIdhZmalOGGYmVkpThhmZlaKE4aZmZXihGFmZqU4YZiZWSn/H4W3szvv1S2XAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "results_for_vis = results_for_vis[results_for_vis[\"Topics\"]==3]\n",
    "#results_for_vis = results_for_vis[results_for_vis[\"Alpha\"]!='asymmetric']\n",
    "#results_for_vis = results_for_vis[results_for_vis[\"Beta\"]!='asymmetric']\n",
    "\n",
    "# now the plot\n",
    "import matplotlib.pyplot as plt\n",
    "#plt.plot(np.array(results_for_vis[\"Alpha\"]), np.array(results_for_vis[\"Coherence\"]))\n",
    "plt.scatter(np.array(results_for_vis[\"Alpha\"]), np.array(results_for_vis[\"Coherence\"]))\n",
    "plt.ylabel(\"C_v Score\")\n",
    "plt.xlabel(\"Alpha\")\n",
    "plt.xticks(np.array(results_for_vis[\"Alpha\"]))\n",
    "plt.show()\n",
    "# 0.81 0.91"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHWZJREFUeJzt3X+wHeV93/H3h3uRjUVISJAcFyEEQp5aAYLtYyG74YcJ2MLMCJxJUuRoYhxi6LhM26jOjBRTUxEaXDRm0iR0CmQ8rRPXFBhXIxcBVjGQ2gFGVxaWLCiWkAnIii2B7cGAK6yrb//YPbp7zz3n3r3S2d1H93xeM3d09zm793y15+x+9/mxzyoiMDMzm8pxTQdgZmbHBicMMzMrxQnDzMxKccIwM7NSnDDMzKwUJwwzMyvFCcPMzEpxwjAzs1KcMMzMrJThpgPop1NOOSUWLFjQdBhmZseULVu2vBwRc6Zab0YljAULFjAyMtJ0GGZmxxRJ/1BmPTdJmZlZKZUnDEnLJD0naZek1V1eXyXpGUnbJD0i6fTCa/MlfU3Ss/k6C6qO18zMuqs0YUgaAu4ALgcWAyskLe5YbSvQiohzgfuB2wqvfRFYFxHvApYA+6qM18zMequ6hrEE2BURuyPiTeAe4MriChHxaES8kS8+CcwDyBPLcERsytd7rbCemZnVrOqEcSrwUmF5T17Wy7XAg/nv7wR+IukrkrZKWpfXWMaRdJ2kEUkj+/fv71vgZmY2XtUJQ13Kuj6xSdJKoAWsy4uGgQuATwPvA84ErpnwxyLuiohWRLTmzJlyVJiZmR2hqhPGHuC0wvI8YG/nSpIuBT4DLI+IA4Vtt+bNWQeB9cB7Ko7XzMx6qDphbAYWSTpD0izgamBDcQVJ7wbuJEsW+zq2PVlSu9pwCfBMxfGamVkPlSaMvGZwA/Aw8Cxwb0TskHSzpOX5auuAE4H7JD0taUO+7ShZc9QjkraTNW/dXWW8ZmbWmyK6dikck1qtVvhObzOz6ZG0JSJaU63nO73NzKwUJwwzMyvFCcPMzEpxwjAzs1KcMMzMrBQnDDMzK8UJw8zMSnHCMDOzUpwwzMysFCcMMzMrxQnDzMxKccIwM7NSnDDMzKwUJwwzMyvFCcPMzEpxwjAzs1KcMMzMrBQnDDMzK8UJw8zMSnHCMDOzUpwwzMysFCcMMzMrxQnDzMxKGW46AJvoxvXb+fJTLzEawZDEivNP45arzmk6LDMbcE4Yiblx/Xb+9skXDy+PRhxedtIwsya5SSoxX37qpWmVm5nVxTWMxIxGTKvcquXmQbMxrmEkZkiaVrlVp9082E7W7ebBG9dvbzgys2Y4YSRmxfmnTavcquPmQbPx3CTVoekmiPZ7uRmkeW4eNBvPCaMglRFKt1x1jhNEAoakrsnBzYM2qCpvkpK0TNJzknZJWt3l9VWSnpG0TdIjkk7veP0kSd+X9FdVx/qlQrIoU24zm5sHzcarNGFIGgLuAC4HFgMrJC3uWG0r0IqIc4H7gds6Xv9T4PEq42zr1dDgBojBdMtV57By6fzDNYohiZVL57v2ZwOr6iapJcCuiNgNIOke4ErgmfYKEfFoYf0ngZXtBUnvBd4OPAS0Ko7VbAI3D5qNqbpJ6lSgOKRkT17Wy7XAgwCSjgM+D/xxZdGZmVlpVSeMbr2DXVt4JK0kq0Wsy4s+BWyMiEnHMEq6TtKIpJH9+/cfVbArl86fVrmZ2SCpuklqD1DsIZwH7O1cSdKlwGeAiyLiQF78fuACSZ8CTgRmSXotIsZ1nEfEXcBdAK1W66i6Gzyk1cysN0WFY8olDQPfBX4T+D6wGfhYROworPNuss7uZRGxs8ffuYasY/yGyd6v1WrFyMhIn6I3a/6+HLM6SNoSEVP2E1faJBURB4EbgIeBZ4F7I2KHpJslLc9XW0dWg7hP0tOSNlQZk1lZnhrEbLxKaxh1cw3D+umM1Q907XAT8L3PXVF3OGaVSaKGYXYs8305ZuM5YZiZWSlOGGZmVooThlkPvi/HbDzPVmvWg+/LMRvPo6TMzAZc2VFSrmGY2THjstsfY+e+1w8vL5o7m02rLq41hnNveohXD4weXj7pLUNsW7us1hjOWvMABwvX+sOCXbdWP9TbfRhmdkzoTBYAO/e9zmW3P1ZbDJ3JAuDVA6Oce9NDtcXQmSwADkZWXjUnDDM7JnQmi6nKq9CZLKYqr0JnspiqvJ/cJGXJ8jxOZmlxwrAkpfJ8dTMb4yYpS9KXn+r+GJRe5TbzLZo7e1rlVTjpLUPTKq/CcLenDE1S3k9OGJak0R7DvXuVm9Vh29plE07Mw6LWUVK9RkN5lFQDbly/nYVrNrJg9QMsXLPRU1k3ZEjdL5d6ldvMl0Kn943rt3cdoVTneaLXe9URgxNGgZ9/kI4V5582rXKzOqTQVNpkDE4YBSl8GSxzy1XnsHLp/MM1iiGJlUvnu8PbGpVCU2mTMThhFKTwZbAxT+1+ZVxt76ndrzQckTUphU7vFJpKm4zBCaMghS+DZVK4q9fSsmnVxROSQ91Tg6TQVNpkDL4Po2DF+aeNG/tfLLd6pdDBaempe96oTinMYNxkDE4YZmbTcMtV5zTel9ZUDG6SKnCnt5lZb04YBe70TkcKHZxmNp4TRkGvrm13edcvhQ5OMxvPfRgFveoRddcvPEtrxsnBLC1OGInxLK1mlio3SRWkcB/Gl7oM652s3MysLk4YBSnclJNKs5iZWSc3SRWkcFOOjXFfjllanDA6pHBTTgqaPlm7L8csPU4YNkEKJ+vJbqJ0whhcTV/IDDr3YSRm5dL50yqvQgp3vPsmSuvk59U0r/KEIWmZpOck7ZK0usvrqyQ9I2mbpEcknZ6XnyfpCUk78tf+edWxpiCF50CkcLJOYcSapSWFC5lBV2nCkDQE3AFcDiwGVkha3LHaVqAVEecC9wO35eVvAL8fEb8GLAP+XNIvVRmvZVI4WacwYs3SksKFzKCruoaxBNgVEbsj4k3gHuDK4goR8WhEvJEvPgnMy8u/GxE789/3AvuAORXH27gUqt1nznnbtMqrkEJNy9KSwoXMoCvd6Z03FS2KiP8t6QRgOCJ+OsVmpwLF+uIe4PxJ1r8WeLDLey8BZgHPl433WJVCZ+/u/W9Mq7wqHrFmRX5eTfNK1TAkfZKsuejOvGgesL7Mpl3KutYfJa0EWsC6jvJ3AH8DfCIiDnXZ7jpJI5JG9u/fXyKktKVQ7U4hBrNOrnU2r2wN41+SNS89BRAROyXNLbHdHqCY/ucBeztXknQp8Bngoog4UCg/CXgAuDEinuz2BhFxF3AXQKvV8hnNbAZzrTPT1PDisgnjQES8qTyzSxqm3GwVm4FFks4Avg9cDXysuIKkd5PVXJZFxL5C+SzgfwJfjIj7SsZ51Joe5y2671i30poZNHufVNmE8bikPwFOkHQZ8Cngq1NtFBEHJd0APAwMAV+IiB2SbgZGImIDWRPUicB9eUJ6MSKWA78LXAj8iqRr8j95TUQ8Xf6/Nz0p3LCWwlxSQ1LX5id3LjbjrDUPcLDwcQwLdt16RXMBDbimLyqb7OcsO0pqNbAf2A5cD2wEbiyzYURsjIh3RsTCiPgPedln82RBRFwaEW+PiPPyn+V5+d9GxPGF8vOqTBaQxjjvFEaCeEhrOjqTBcDByMqtfimMYmyyj3HKGkZ+L8V/i4iVwN2VR9SgFDp7UxgJ4kkYxzR9NdmZLKYqt2qlMIqxyRaAKRNGRIxKmiNpVn4vxYyVQlNMKidrdy6m0USZiqYTZyoG/aKybB/GC8A3JW0AXm8XRsTtVQTVlNmzjuPVA6Ndy61+TZ+kUriaTEFKibPp70QKg1KavKgsmzD25j/HAb9QXTjN6pYsJiuvQkoHZ5NS2A8pXE0Oq3vz03CNZ6huV7Pt8kGb8j6FQSnQXAtAqUvniFgbEWuB24HPF5atz1LoeIfs4Fy4ZiMLVj/AwjUba58RNIX9kMIAhF23XjEhOQzqKKkUvhODrlQNQ9LZZHdb/3K+/DLZxIA7KoxtIKVwVZvClVwK+yGFAQgwmMmhmxS+Eyn0czapbOP8XcCqiDg9Ik4H/i0zcMTUormzp1VehRSuar/UowmiV3kVUtgPlum1x+v+JFL4Tgz6kPOyCWN2RDzaXoiIx4D6zqI12bTq4gnJYdHc2WxadXFtMaTwhUyhnTaFGXPdBJL5vR4P7+pVXpUUjo1U5rNqqsm4bKf3bkn/jqxZCmAl8L1qQmpWncmhm1SG1TYthRlzU2gCSUEq38mU4mjyeDwWpgb5A2At8JV8+e+AT1QSkTX+hUxBCifrQW+vTpGPjWaHe5dKGBHxY+BfVRqJHdb0WPNFc2ezc9/rXcvrksLJ+sw5b+u6H+psFktBCoMgbEyTF1Nln4exqfh4VEknS3q4urAGVwpz1aTQHJRCe3UK+yEF7stJS5Od/2U7vU+JiJ+0F/IaR5nnYdg0pXBwptAclELnYgr7IQXeD2lp8mKqbB/GIUnzI+JFOPy4Vn9bKpDCwZlCcxA0316dyn4wKzoWpgb5DPANSY/nyxcC11UTkjUtlRvWmpbKfmi6T8vS09TFVNlO74ckvQdYmhf9UUS8XF1YzfHBaW0pDONMocPZNS1rm7QPQ9Lpkn4RIE8QrwOXAb+fP0J1RkmhwzmFu2pT6EdJxS1XncPzt36EFz53Bc/f+pHaLx5S+CxSGIBgaZiq0/te8ju6JZ0H3Ae8CPw68J+rDa1+KRycKdxlnUI/imVS+CxSGIBgaZiqSeqEiNib/76S7Jncn5d0HFDp41KbkMLBmYIU5vy3tDQ9AMHSMFUNo3iOuAR4BCAiDlUWUYNSmNwshRhSqOWYWXqmShhfl3SvpP8EnAx8HUDSO4AZ97jWFNpqU4jB0pHCBYRZ21QJ49+QzR/1AvAbEfHzvPxXyYbazigptNWmEINPUunwBYSlZNI+jIgI4J4u5Vsri8gaby9O5f4DS2NoL3i4uWUUM6hDt9VqxcjIyBFv3znmvW0QR4T4BGFtPi5mPklbIqI15XpOGGMWrtnY8wal52/9yNGEZnbM8nEx85VNGGWf6f1RYGNEHDjqyBLmYbWWoqZrez4urK3sbLXLge9K+htJV0gqOwfVMcWdvZaaFGYf8HFhbaUSRkR8AjiL7E7vjwHPS/rrKgNrgkekWGpSmH3Ax4W1la1hkA+pfZBs1NQW4MqqgmpKCkNazYpSaA665apzGO6oTAzLT9sbRGX7MJYBVwMfBB4D/hr43erCak7TQ1pT0XS7uWPIpDBNy7k3PcTBjiAORla+be2yGiOxppXti7iGrGZx/Uzv+LY0ptR2DJkUpml59cDotMpt5irbh3F1RKzvlSwkPdFrW0nLJD0naZek1V1eXyXpGUnbJD2SP82v/drHJe3Mfz5eJlY7eim0mzsGs/SU7sOYwlu7FUoaAu4ALgcWAyskLe5YbSvQiohzgfuB2/Jtfxm4CTgfWALcJOnkPsVrk0ih3dwxZDxCyVLSr4TR6whaAuyKiN0R8SZZs9a4zvKIeDQi3sgXnwTm5b9/GNgUET+KiB8DmwA3mNYghZOUY8ikMELppLcMTavcZq5+JYxeTgWK9fc9eVkv15KNxDqSba1PUjhJOYZMCiP3tq1dNiE5nPSWIXd4D6B+3YA3nSeLdq2NSFoJtICLprOtpOuA6wDmz58/ZaA2tRQmvHMM4+NoeuSek4PBFHNJSToLeHtEfLOj/AJgb0Q8ny+fHRHf6bL9+4F/HxEfzpfXAETErR3rXQr8JXBRROzLy1YAF0fE9fnyncBjEfHlXvEe7VxSZpa2poc5z9QYys4lNVWT1J8DP+1S/rP8NQC6JYvcZmCRpDMkzSK7l2NDR6DvBu4ElreTRe5h4EOSTs47uz+Ul5nZAEphmpRBj2GqhLEgIrZ1FkbECLBgqj8eEQeBG8hO9M8C90bEDkk3S1qer7YOOBG4T9LTkjbk2/4I+FOypLMZuDkvMxsoN67fzsI1G1mw+gEWrtlY68kpJSkMcx70GKbqw+g6XDZ3Qpk3iIiNwMaOss8Wfr90km2/AHyhzPuYzUQp3DyYihSGOQ96DFMljM2SPhkRdxcLJV1LNp+UWWVmYlvxdE12NTloCWNI6vlcDsdQTwxlnun9CUmPSfp8/vM48IfAv648OhtYg95W3JbCFW0qUhjmPOgxTJowIuKHEfEBYC3wQv6zNiLeHxE/qDw6G1iD3lbclsLNg2ZtZeeSejQi/jL/+XrVQVnzmu5oTeHKOoUYUriiTUUKCXzQY5iRT86zo5NCR+ugtxW3pXLzYApSSOCDHoMThk2QQkfrivNPG5e0iuV1SSEGSONO7xSkkMAHPQYnDJsghauoFK6sU4ghFZfd/hg7971+eHnR3NlsWnVxrTGkkMAHPQYnDJsghasoSOPKOoUYmtaZLAB27nudy25/rNak8dTuV6ZVPlNjaPJCxgnDJkjhKsrS0ZkspiqfyXGkEAM0dyHjhGETuCnGzLpxwrCuUmiKafouazMbr+oHKJkdkRTusrbMormzp1U+k+NIIYYmOWFYklK4Qcoym1ZdPOGE2MQoqRTiSCGGJrlJypKUwtBeG5PKCTGFOFKIoSmuYViSPIeSWXqcMCxJnkPJLD1ukrIkeWivWXoUM6hNuNVqxcjISNNhmJkdUyRtiYjWVOu5ScrMzEpxwjAzs1KcMMzMrBQnDDMzK8UJw8zMSvGwWrNJpDABYgoxmIEThllPKTzbPIUYzNrcJGXJunH9dhau2ciC1Q+wcM3G2meqTWECxBRiMGtzDcOSlMKVdQoTIKYQg1mbaxiWpBSurFOYADGFGMzanDAsSSlcWacwAWIKMZi1uUnKkjQkdU0OdV5ZpzABYgoxmLV58kFLUmcfRtvKpfN9sjTrs2QmH5S0TNJzknZJWt3l9QslfUvSQUm/3fHabZJ2SHpW0l9IbrgdFLdcdQ4rl84/XKMYkpwszBpWaZOUpCHgDuAyYA+wWdKGiHimsNqLwDXApzu2/QDwz4Bz86JvABcBj1UZs6XjlqvOcYIwS0jVfRhLgF0RsRtA0j3AlcDhhBERL+SvHerYNoC3ArMAAccDP6w4XjMz66HqhHEqUBwHuQc4v8yGEfGEpEeBfyRLGH8VEc/2P0Qzs2NLU9PFVN2H0a3PoVQvu6SzgHcB88gSzyWSLuyy3nWSRiSN7N+//6iCNTNLXXtASHsUYfum1jpmQqg6YewBigPG5wF7S277UeDJiHgtIl4DHgSWdq4UEXdFRCsiWnPmzDnqgM3MUtbkTa1VJ4zNwCJJZ0iaBVwNbCi57YvARZKGJR1P1uHtJikzG2hN3tRaacKIiIPADcDDZCf7eyNih6SbJS0HkPQ+SXuA3wHulLQj3/x+4HlgO/Bt4NsR8dUq4zUzS12T08VUfqd3RGwENnaUfbbw+2aypqrO7UaB66uOz8zsWLLi/NO63tRax3QxnhrEzOwY0uR0MZ4axMxswCUzNYiZmc0MThhmZlaKE4aZmZXiTm+zSTQ1BYNZipwwzHpI4bniZilxk5RZDyk8V9wsJU4YZj2k8Fxxs5Q4YZj10OQUDGYpcsIw66HXVAt1TMFgliJ3epv10OQUDGYp8tQgZmYDzlODmJlZXzlhmJlZKU4YZmZWihOGmZmV4oRhZmalOGGYmVkpThhmZlaKE4aZmZXihGFmZqU4YZiZWSlOGGZmVooThpmZleKEYWZmpThhmJlZKU4YZmZWihOGmZmV4oRhZmalOGGYmVkpThhmZlZK5QlD0jJJz0naJWl1l9cvlPQtSQcl/XbHa/MlfU3Ss5KekbSg6njNzKy7ShOGpCHgDuByYDGwQtLijtVeBK4B/nuXP/FFYF1EvAtYAuyrLlozM5vMcMV/fwmwKyJ2A0i6B7gSeKa9QkS8kL92qLhhnliGI2JTvt5rFcdqZmaTqLpJ6lTgpcLynrysjHcCP5H0FUlbJa3LayzjSLpO0oikkf379/chZDMz66bqhKEuZVFy22HgAuDTwPuAM8marsb/sYi7IqIVEa05c+YcaZxmZjaFqhPGHuC0wvI8YO80tt0aEbsj4iCwHnhPn+MzM7OSqk4Ym4FFks6QNAu4GtgwjW1PltSuNlxCoe/DzMzqVWnCyGsGNwAPA88C90bEDkk3S1oOIOl9kvYAvwPcKWlHvu0oWXPUI5K2kzVv3V1lvGZm1psiynYppK/VasXIyEjTYZiZHVMkbYmI1lTr+U5vMzMrxQnDzMxKccIwM7NSnDDMzKwUJwwzMytlRo2SkrQf+Ic+/bn3Alv69Lccg2NwDP2VQhwzKYbTI2LKqTJmVMLoJ0kREd2mNnEMjsExNCyFOAYxBjdJmZlZKU4YZmZWihNGbym01TmGjGPIOIYxKcQxcDG4D8PMzEpxDcPMzEoZuIQhaVRS5D+Hurz+ycLrkQ/VRdI3iuUVx3Cw8Ppox2t1xXCoYz/8sN8xHGEcb+TltX0eXdb7ed0xdNkPhwqv1bYfOuOoO4aOfdBUDP/YEcNoXl7n9+GHHTG8WHitb8dnp4FKGJLeRfZ//hHwsawoOwEV3AmQD1UL4JS8fDMwylG2GZaM4cfAgR5/4lD+U3UMAfxZYT/M7WcM04jjf0SE8jgOASfk5XV+Hkj6r0w8XmqNAbLvZf5TjKWW74SkV8keM/C1/PPYXncMhf9/+3tZVNex8atZKCHgZ4x9L+r8PswtxHCI8Q+q68vx2TW2QerDkHQQGGqPW25n7uLBl2fl0YgYlnQB8HfAcZHvqHwbHenY5zIxdMRyKCKGOspH85gqjyF//U3g+OL7HW0MRxjHhH1f1+fR/izIDuSDEXF8nTFM9R51fCfyfRCTfD51fy8nHB9174fO9QvbVP19COCliJgvSWT7oa/HZzcDVcOg+zPGu5XtBYiI/5MvP9lADFWabgzHU81ojFJxtJtB8td+WncMecKkM3HXGUMhlnZzQ9lHHfc7BnU2xTQQAxprIj27gRh2ke8HYAj4eQMxQPbIa4CDZAF9o89xTDBoCeNI9fsLccwotJ9e31QMEdG+UgrgF+p8b2WPCD4eeLnO9+3i74HrCvvhHQ3GcgXwCnCcpAcaimEOQEQ828B7L2SsOWgUOF5jj5Kuy/cYS1rt8/i+qt900BJGt6vkbmX/BCBvkgK4oMs6VcdQpVIx5FeQArZHRBWPx53uvnh7AzHcmf97SqETcVh5x3dNMRARv1H4DH6tj+9dOoaxUGJjRLT79j7cQAyQfS/7XcMpG4OAHwBExHBe9kidMUTEmYW+nP+Sl/1WH2PoatASxgcBJL0saQXZB9/ZudyuZgI8DtkRUnMMVZsyhrwZ5jjgpxFxboNxvFpY/EHdMUTEb3V0tEJHH0bVMeSv7SwsfqeP7106BrI+nHa7+lN5WT+ba0sdG5Jey3/t52cwrRjIOr6R9Hq+/Ot1xiDpqUKt5l/08b0nNVCd3jDWGZQvtjuuDneeSfoj4PbCJj+KiF/Jt+3cWT+LiLdVEMNBxpJWe6X2gTphZMiRtK2XiGHCF6PfMRxhHP8vIk7oEUcln0fHukEhYdT4neh8nzciYnaPGKr6TrRH5FBcp84Yiu/V2aFb4354g7HReuPep8bvQ/F1yJor7+4RwxEfnxPiGrSEYWZmR2bQmqTMzOwIOWGYmVkpThhmZlaKE4aZmZXihGFmZqU4YZgdBWWzij4t6duSviXpA1Os/0uSPlVXfGb95GG1ZkdB0msRcWL++4eBP4mIiyZZfwHwvyKi33MgmVXONQyz/jmJbGp6ACT9saTNkrZJWpsXfw5YmNdK1kk6UdIjee1ku6QrG4ncrIThqVcxs0mcIOlp4K1kEwJeAiDpQ8AiYAnZ1A4bJF0IrAbOjojz8vWGgY9GxKuSTgGelLShz9PRmPWFE4bZ0flZ4eT/fuCLks4GPpT/bM3XO5EsgbzYsb2AP8uTySHgVLJJFquYN8vsqDhhmPVJRDyR1xLmkCWCWyPizuI6eR9G0e/l6783In4u6QWy2opZctyHYdYnkv4p2aSRrwAPA38gqd0hfqqkuWQPgCo+0+MXgX15svggcHrNYZuV5hqG2dFp92FAVqv4eESMAl9T9mzmJ7JJXnkNWBkRz0v6pqTvAA8C/xH4qqQR4Gng/9b/XzArx8NqzcysFDdJmZlZKU4YZmZWihOGmZmV4oRhZmalOGGYmVkpThhmZlaKE4aZmZXihGFmZqX8fww886i0UAA/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "results_for_vis = pd.read_csv(\"lda_tuning_results_overall.csv\")\n",
    "results_for_vis = results_for_vis[results_for_vis[\"Topics\"]==3]\n",
    "#results_for_vis = results_for_vis[results_for_vis[\"Alpha\"]!='asymmetric']\n",
    "#results_for_vis = results_for_vis[results_for_vis[\"Beta\"]!='symmetric']\n",
    "#results_for_vis = results_for_vis[results_for_vis[\"Beta\"]=='0.61']\n",
    "\n",
    "# now the plot\n",
    "import matplotlib.pyplot as plt\n",
    "#plt.plot(np.array(results_for_vis[\"Beta\"]), np.array(results_for_vis[\"Coherence\"]))\n",
    "plt.scatter(np.array(results_for_vis[\"Beta\"]), np.array(results_for_vis[\"Coherence\"]))\n",
    "plt.ylabel(\"C_v Score\")\n",
    "plt.xlabel(\"Beta\")\n",
    "plt.xticks(np.array(results_for_vis[\"Beta\"]))\n",
    "plt.show()\n",
    "#0.81 0.91"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.7/site-packages/pyLDAvis/_prepare.py:257: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  return pd.concat([default_term_info] + list(topic_dfs))\n"
     ]
    }
   ],
   "source": [
    "# k = 3, a = b = 0.81\n",
    "\n",
    "# supporting function\n",
    "corpus=doc_term_matrix\n",
    "id2word=dictionary\n",
    "lda_model = gensim.models.LdaMulticore(corpus=corpus,\n",
    "                                           id2word=id2word,\n",
    "                                           num_topics=3, \n",
    "                                           random_state=100,\n",
    "                                           chunksize=100,\n",
    "                                           passes=10,\n",
    "                                           alpha=0.61,\n",
    "                                           eta=0.41)\n",
    "# Visualize the topics\n",
    "pyLDAvis.enable_notebook()\n",
    "vis = pyLDAvis.gensim.prepare(lda_model, doc_term_matrix, dictionary)\n",
    "vis\n",
    "pyLDAvis.save_html(vis, 'BB_overall_sep_3.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>star</th>\n",
       "      <th>sku</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>bagit hold candy not included trick treat you ...</td>\n",
       "      <td>5</td>\n",
       "      <td>8566349</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>the bag would great deal available for store p...</td>\n",
       "      <td>1</td>\n",
       "      <td>8566349</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>loved the movie and played delivered ontime</td>\n",
       "      <td>5</td>\n",
       "      <td>6083463</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>guy cant say headoverheels for dirty dancing b...</td>\n",
       "      <td>3</td>\n",
       "      <td>6083463</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>this movie great for all age shipping extremel...</td>\n",
       "      <td>5</td>\n",
       "      <td>8977805</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>154</td>\n",
       "      <td>bought this christmas gift for family with fou...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>155</td>\n",
       "      <td>our purchase christmas gift for our granddaugh...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>156</td>\n",
       "      <td>this just quick note for anyone who concerned ...</td>\n",
       "      <td>4</td>\n",
       "      <td>3792604</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>157</td>\n",
       "      <td>stargate the entire series just amazing able w...</td>\n",
       "      <td>5</td>\n",
       "      <td>8553399</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>158</td>\n",
       "      <td>very excited receive this set however once beg...</td>\n",
       "      <td>1</td>\n",
       "      <td>8553399</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>159 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               content  star      sku  label\n",
       "0    bagit hold candy not included trick treat you ...     5  8566349      1\n",
       "1    the bag would great deal available for store p...     1  8566349      0\n",
       "2          loved the movie and played delivered ontime     5  6083463      1\n",
       "3    guy cant say headoverheels for dirty dancing b...     3  6083463      0\n",
       "4    this movie great for all age shipping extremel...     5  8977805      1\n",
       "..                                                 ...   ...      ...    ...\n",
       "154  bought this christmas gift for family with fou...     5  3792604      0\n",
       "155  our purchase christmas gift for our granddaugh...     5  3792604      1\n",
       "156  this just quick note for anyone who concerned ...     4  3792604      1\n",
       "157  stargate the entire series just amazing able w...     5  8553399      0\n",
       "158  very excited receive this set however once beg...     1  8553399      1\n",
       "\n",
       "[159 rows x 4 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Coherence Score:  0.2428341521104599\n"
     ]
    }
   ],
   "source": [
    "# Compute Coherence Score\n",
    "coherence_model_lda = CoherenceModel(model=lda_model,corpus=doc_term_matrix, texts=tokenized_overall,coherence='c_v')\n",
    "coherence_lda = coherence_model_lda.get_coherence()\n",
    "print('\\nCoherence Score: ', coherence_lda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus=doc_term_matrix\n",
    "id2word=dictionary\n",
    "model_cnt = [0,0,0]\n",
    "temp_1,temp_2,temp_3 = 0, 0, 0\n",
    "cnt = 0\n",
    "for i in range(len(overall_reviews)):\n",
    "    corpus_1= [id2word.doc2bow(word_tokenize(overall_reviews[i]))]\n",
    "    temp_model = lda_model.get_document_topics(corpus_1,minimum_probability=1e-4)\n",
    "    if (len(temp_model[0])>2):\n",
    "        temp_1 += temp_model[0][0][1]\n",
    "        temp_2 += temp_model[0][1][1]\n",
    "        temp_3 += temp_model[0][2][1]\n",
    "        cnt += 1\n",
    "    else:\n",
    "        print(i)        \n",
    "model_cnt[0] = temp_1/cnt # topic 1\n",
    "model_cnt[1] = temp_2/cnt # topic 2\n",
    "model_cnt[2] = temp_3/cnt # topic 3        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "159"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(overall_reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "first dear best buy youve got dvd content wrong disk doe contain episode featuring afterworld tournament saiyaman sagabr contains quotcell returnsquot horror wont end save world gokus decision one wish free future doe contain episode list description cell returnsquot folk picky sort thing fan buying disk dedicated series want know theyre buyingbr second thanks lovely lady best buy honored competitor price wasnt aware given proof lower price best buy would indeed honor competitor price guy rockbr onto disk new set excellent upgrade originally aired episode season six exception theyre uncut restored remasteredand bonusuthe disk contain original toonami run dbz well japanese language edition many company willing include three version product theyve edit order meet american television guideline original japanese product theyve worked nice see company make effort please fan tasteswhile many adult otaku appreciate uncutrestored edition parent appreciate edited version younger uncorrupted fan enjoy japan raw ill always fan dub kudos funimation going back making script change best reflected storyline made better localization series youve made loving dbz cool\n",
      "[(0, 0.9832913)]\n"
     ]
    }
   ],
   "source": [
    "corpus_1= [id2word.doc2bow(word_tokenize(overall_reviews[132]))]\n",
    "temp_model = lda_model[corpus_1]\n",
    "print(overall_reviews[132])\n",
    "for topic in temp_model:\n",
    "    print(topic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0000075752811752"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(model_cnt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>star</th>\n",
       "      <th>sku</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>bagit hold candy not included trick treat you ...</td>\n",
       "      <td>5</td>\n",
       "      <td>8566349</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>the bag would great deal available for store p...</td>\n",
       "      <td>1</td>\n",
       "      <td>8566349</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>loved the movie and played delivered ontime</td>\n",
       "      <td>5</td>\n",
       "      <td>6083463</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>guy cant say headoverheels for dirty dancing b...</td>\n",
       "      <td>3</td>\n",
       "      <td>6083463</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>this movie great for all age shipping extremel...</td>\n",
       "      <td>5</td>\n",
       "      <td>8977805</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>154</td>\n",
       "      <td>bought this christmas gift for family with fou...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>155</td>\n",
       "      <td>our purchase christmas gift for our granddaugh...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>156</td>\n",
       "      <td>this just quick note for anyone who concerned ...</td>\n",
       "      <td>4</td>\n",
       "      <td>3792604</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>157</td>\n",
       "      <td>stargate the entire series just amazing able w...</td>\n",
       "      <td>5</td>\n",
       "      <td>8553399</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>158</td>\n",
       "      <td>very excited receive this set however once beg...</td>\n",
       "      <td>1</td>\n",
       "      <td>8553399</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>159 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               content  star      sku  label\n",
       "0    bagit hold candy not included trick treat you ...     5  8566349      1\n",
       "1    the bag would great deal available for store p...     1  8566349      0\n",
       "2          loved the movie and played delivered ontime     5  6083463      1\n",
       "3    guy cant say headoverheels for dirty dancing b...     3  6083463      0\n",
       "4    this movie great for all age shipping extremel...     5  8977805      1\n",
       "..                                                 ...   ...      ...    ...\n",
       "154  bought this christmas gift for family with fou...     5  3792604      0\n",
       "155  our purchase christmas gift for our granddaugh...     5  3792604      1\n",
       "156  this just quick note for anyone who concerned ...     4  3792604      1\n",
       "157  stargate the entire series just amazing able w...     5  8553399      0\n",
       "158  very excited receive this set however once beg...     1  8553399      1\n",
       "\n",
       "[159 rows x 4 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>star</th>\n",
       "      <th>sku</th>\n",
       "      <th>label</th>\n",
       "      <th>topic 0</th>\n",
       "      <th>topic 1</th>\n",
       "      <th>topic 2</th>\n",
       "      <th>topic 3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>bagit hold candy not included trick treat you ...</td>\n",
       "      <td>5</td>\n",
       "      <td>8566349</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.035076</td>\n",
       "      <td>0.038941</td>\n",
       "      <td>0.925987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>the bag would great deal available for store p...</td>\n",
       "      <td>1</td>\n",
       "      <td>8566349</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.031136</td>\n",
       "      <td>0.926000</td>\n",
       "      <td>0.042885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>loved the movie and played delivered ontime</td>\n",
       "      <td>5</td>\n",
       "      <td>6083463</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.533636</td>\n",
       "      <td>0.219605</td>\n",
       "      <td>0.246712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>guy cant say headoverheels for dirty dancing b...</td>\n",
       "      <td>3</td>\n",
       "      <td>6083463</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.032270</td>\n",
       "      <td>0.937522</td>\n",
       "      <td>0.030212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>this movie great for all age shipping extremel...</td>\n",
       "      <td>5</td>\n",
       "      <td>8977805</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.097499</td>\n",
       "      <td>0.174463</td>\n",
       "      <td>0.728103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>154</td>\n",
       "      <td>bought this christmas gift for family with fou...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.018154</td>\n",
       "      <td>0.953994</td>\n",
       "      <td>0.027857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>155</td>\n",
       "      <td>our purchase christmas gift for our granddaugh...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.023886</td>\n",
       "      <td>0.949697</td>\n",
       "      <td>0.026427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>156</td>\n",
       "      <td>this just quick note for anyone who concerned ...</td>\n",
       "      <td>4</td>\n",
       "      <td>3792604</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.049815</td>\n",
       "      <td>0.888806</td>\n",
       "      <td>0.061348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>157</td>\n",
       "      <td>stargate the entire series just amazing able w...</td>\n",
       "      <td>5</td>\n",
       "      <td>8553399</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.024106</td>\n",
       "      <td>0.950166</td>\n",
       "      <td>0.025714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>158</td>\n",
       "      <td>very excited receive this set however once beg...</td>\n",
       "      <td>1</td>\n",
       "      <td>8553399</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.018526</td>\n",
       "      <td>0.962070</td>\n",
       "      <td>0.019398</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>159 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               content  star      sku  label  \\\n",
       "0    bagit hold candy not included trick treat you ...     5  8566349      1   \n",
       "1    the bag would great deal available for store p...     1  8566349      0   \n",
       "2          loved the movie and played delivered ontime     5  6083463      1   \n",
       "3    guy cant say headoverheels for dirty dancing b...     3  6083463      0   \n",
       "4    this movie great for all age shipping extremel...     5  8977805      1   \n",
       "..                                                 ...   ...      ...    ...   \n",
       "154  bought this christmas gift for family with fou...     5  3792604      0   \n",
       "155  our purchase christmas gift for our granddaugh...     5  3792604      1   \n",
       "156  this just quick note for anyone who concerned ...     4  3792604      1   \n",
       "157  stargate the entire series just amazing able w...     5  8553399      0   \n",
       "158  very excited receive this set however once beg...     1  8553399      1   \n",
       "\n",
       "     topic 0   topic 1   topic 2   topic 3  \n",
       "0          0  0.035076  0.038941  0.925987  \n",
       "1          0  0.031136  0.926000  0.042885  \n",
       "2          0  0.533636  0.219605  0.246712  \n",
       "3          0  0.032270  0.937522  0.030212  \n",
       "4          0  0.097499  0.174463  0.728103  \n",
       "..       ...       ...       ...       ...  \n",
       "154        0  0.018154  0.953994  0.027857  \n",
       "155        0  0.023886  0.949697  0.026427  \n",
       "156        0  0.049815  0.888806  0.061348  \n",
       "157        0  0.024106  0.950166  0.025714  \n",
       "158        0  0.018526  0.962070  0.019398  \n",
       "\n",
       "[159 rows x 8 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BB['topic 0'] = 0\n",
    "BB['topic 1'] = np.nan\n",
    "BB['topic 2'] = np.nan\n",
    "BB['topic 3'] = np.nan\n",
    "for i in range(len(overall_reviews)):\n",
    "    corpus_1= [id2word.doc2bow(word_tokenize(overall_reviews[i]))]\n",
    "    temp_model = lda_model.get_document_topics(corpus_1,minimum_probability=1e-4)\n",
    "    BB.at[i, 'topic 1'] = temp_model[0][0][1]\n",
    "    BB.at[i, 'topic 2'] = temp_model[0][1][1]\n",
    "    BB.at[i, 'topic 3'] = temp_model[0][2][1]\n",
    "BB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>star</th>\n",
       "      <th>sku</th>\n",
       "      <th>label</th>\n",
       "      <th>topic 0</th>\n",
       "      <th>topic 1</th>\n",
       "      <th>topic 2</th>\n",
       "      <th>topic 3</th>\n",
       "      <th>area</th>\n",
       "      <th>GINI coe</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>bagit hold candy not included trick treat you ...</td>\n",
       "      <td>5</td>\n",
       "      <td>8566349</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.035076</td>\n",
       "      <td>0.038941</td>\n",
       "      <td>0.925987</td>\n",
       "      <td>2.436382</td>\n",
       "      <td>0.593936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>the bag would great deal available for store p...</td>\n",
       "      <td>1</td>\n",
       "      <td>8566349</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.031136</td>\n",
       "      <td>0.926000</td>\n",
       "      <td>0.042885</td>\n",
       "      <td>5.953133</td>\n",
       "      <td>0.007810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>loved the movie and played delivered ontime</td>\n",
       "      <td>5</td>\n",
       "      <td>6083463</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.533636</td>\n",
       "      <td>0.219605</td>\n",
       "      <td>0.246712</td>\n",
       "      <td>7.147409</td>\n",
       "      <td>-0.191236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>guy cant say headoverheels for dirty dancing b...</td>\n",
       "      <td>3</td>\n",
       "      <td>6083463</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.032270</td>\n",
       "      <td>0.937522</td>\n",
       "      <td>0.030212</td>\n",
       "      <td>6.008256</td>\n",
       "      <td>-0.001377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>this movie great for all age shipping extremel...</td>\n",
       "      <td>5</td>\n",
       "      <td>8977805</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.097499</td>\n",
       "      <td>0.174463</td>\n",
       "      <td>0.728103</td>\n",
       "      <td>3.477978</td>\n",
       "      <td>0.420336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>154</td>\n",
       "      <td>bought this christmas gift for family with fou...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.018154</td>\n",
       "      <td>0.953994</td>\n",
       "      <td>0.027857</td>\n",
       "      <td>5.961216</td>\n",
       "      <td>0.006463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>155</td>\n",
       "      <td>our purchase christmas gift for our granddaugh...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.023886</td>\n",
       "      <td>0.949697</td>\n",
       "      <td>0.026427</td>\n",
       "      <td>5.989891</td>\n",
       "      <td>0.001684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>156</td>\n",
       "      <td>this just quick note for anyone who concerned ...</td>\n",
       "      <td>4</td>\n",
       "      <td>3792604</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.049815</td>\n",
       "      <td>0.888806</td>\n",
       "      <td>0.061348</td>\n",
       "      <td>5.953688</td>\n",
       "      <td>0.007718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>157</td>\n",
       "      <td>stargate the entire series just amazing able w...</td>\n",
       "      <td>5</td>\n",
       "      <td>8553399</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.024106</td>\n",
       "      <td>0.950166</td>\n",
       "      <td>0.025714</td>\n",
       "      <td>5.993484</td>\n",
       "      <td>0.001085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>158</td>\n",
       "      <td>very excited receive this set however once beg...</td>\n",
       "      <td>1</td>\n",
       "      <td>8553399</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.018526</td>\n",
       "      <td>0.962070</td>\n",
       "      <td>0.019398</td>\n",
       "      <td>5.996472</td>\n",
       "      <td>0.000587</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>159 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               content  star      sku  label  \\\n",
       "0    bagit hold candy not included trick treat you ...     5  8566349      1   \n",
       "1    the bag would great deal available for store p...     1  8566349      0   \n",
       "2          loved the movie and played delivered ontime     5  6083463      1   \n",
       "3    guy cant say headoverheels for dirty dancing b...     3  6083463      0   \n",
       "4    this movie great for all age shipping extremel...     5  8977805      1   \n",
       "..                                                 ...   ...      ...    ...   \n",
       "154  bought this christmas gift for family with fou...     5  3792604      0   \n",
       "155  our purchase christmas gift for our granddaugh...     5  3792604      1   \n",
       "156  this just quick note for anyone who concerned ...     4  3792604      1   \n",
       "157  stargate the entire series just amazing able w...     5  8553399      0   \n",
       "158  very excited receive this set however once beg...     1  8553399      1   \n",
       "\n",
       "     topic 0   topic 1   topic 2   topic 3      area  GINI coe  \n",
       "0          0  0.035076  0.038941  0.925987  2.436382  0.593936  \n",
       "1          0  0.031136  0.926000  0.042885  5.953133  0.007810  \n",
       "2          0  0.533636  0.219605  0.246712  7.147409 -0.191236  \n",
       "3          0  0.032270  0.937522  0.030212  6.008256 -0.001377  \n",
       "4          0  0.097499  0.174463  0.728103  3.477978  0.420336  \n",
       "..       ...       ...       ...       ...       ...       ...  \n",
       "154        0  0.018154  0.953994  0.027857  5.961216  0.006463  \n",
       "155        0  0.023886  0.949697  0.026427  5.989891  0.001684  \n",
       "156        0  0.049815  0.888806  0.061348  5.953688  0.007718  \n",
       "157        0  0.024106  0.950166  0.025714  5.993484  0.001085  \n",
       "158        0  0.018526  0.962070  0.019398  5.996472  0.000587  \n",
       "\n",
       "[159 rows x 10 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The y values.  A numpy array is used here,\n",
    "# but a python list could also be used.\n",
    "fair_y = [0,0.333333,0.333333,0.333333]\n",
    "fair_y_cum = np.cumsum(fair_y)\n",
    "fair_area = np.trapz(fair_y_cum, dx=4)\n",
    "BB['area'] = np.nan\n",
    "BB['GINI coe'] = np.nan\n",
    "for i in range(len(overall_reviews)):\n",
    "    if i != 132:\n",
    "        y = []\n",
    "        for j in range(4):\n",
    "            temp = 'topic '+str(j)\n",
    "            y.append(BB.at[i,temp])\n",
    "            # [0, 0.0532762, 0.06006141, 0.8866621]\n",
    "        y_cum = np.cumsum(y)\n",
    "        # [0.  0.05327629 0.11333771 0.99999983]\n",
    "        BBarea = np.trapz(y_cum, dx=4)\n",
    "        BB.at[i, 'area'] = BBarea\n",
    "        BB.at[i, 'GINI coe'] = (fair_area - BBarea)/fair_area\n",
    "        \n",
    "BB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbQAAAEWCAYAAAAO4GKjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xd4FNX6wPHvS0LvvQUICAiBhE4AkaYUFaVLs4CIVwX9IcrFa+9XsV27F1SCgKDiFSwoSpdeBCH0AAFCCzUkhJB2fn+cSVxC+ibZJLyf58mT3ZnZM+/Ozsw7c2bmHDHGoJRSShV0RTwdgFJKKZUTNKEppZQqFDShKaWUKhQ0oSmllCoUNKEppZQqFDShKaWUKhQ0oeUQESkpIj+KSISIfJuH871ZRP4qCGWLyFIRGeq8flBEFudg2WNE5MecKi8/EJHXReQzT8eRk0SkvIhEiUjlHCyzlYjscMq9N6fKLWhEpIKIGBGp4ulYXInIHBF5NC/mlamEJiIjRGSTs8IcF5FfRKRzimlGOQvzzhTDu4lIovPZSBHZIyKj05lXORH5j4gcdj4T4rzPVz9SKgYD1YHKxpghriNE5FPnu0SJSKyIxLm8/8WdmRpjFhtjWmT1c87OMs75TSJFZLeIvCci1bJadmZ3vMaYHsaYr7MaayrzayIi8SnK/twYc7u7ZaeYTwMRiRcRn1TG/SIirzivB4vINhG5ICKnROT31D6T4vPFXdaBKGcbiXZ5Pygnv0tWOAcHSXFcctl+o0TktDtlG2MijDFljDFncipe4BngG6fcGTlVqIhMcPZpvTKYrpSIfCwix5xtab+IvJxTceQF57vGO79xhLO/75ETZRtjhhtj3s+JsjKSYUITkYnAf4DXsDvsusDHQL8Uk94LnHX+p3TMGFMGKAdMBqaJiF8q8yoGLAGaAX2c6TsBZ4D2mftKV5TnndXPuKEesNcYE59yhDHmQWdjK4Ndjl8nvTfG3JKHMaY0wxhTFqgMDAF8gU0iUjUnZyIiRUSkwNUGGGMOAKuBu1yHi0h14GbgS2c9/gwYD5QHrgOmAYkZlH3ZZR0oA4QDvVyGfZeVWHNyXXcODpLiGgAccIkrPx5Y1gN2ZOeDGSy39PZprl7FbjstsPus3sDO7MTjYb86v3kl4BtgnrNPLjiMMWn+YTfQKGBIBtPVw27Ag4B4oLrLuG5AWIrpTwGDUynnfuAkUCadeRmgocv7IOAV13lhk+YJYCawC+jrMr03cBpo7bzvAKwBzgN/Ad3SmXdTYLkz7Q7gDmf4i0AsEOcsrzHplPECMCuV4YOwG8F5YDHQyGXcCeCfwG7sBjYVKO6M6wOEuEzrCyxwvuNp4O004ngd+CzFsKLO8noljbKfBY4DF5zpbgT6p/juG5xp1wEvAeuBGMDHGXaXM/5BYCnwX6e8nUCXFN+5c2rxYnf+xplfFNDKKW+xy/RdgT+BCGe+7VzGrQOed/5fABYCFdNYTvcBO1MMmwiscV7fBaxLb/vIzF/K7+vynWcDc4BIYBvQMsVnnnDWxWhnWB2X3/8A8KDL9F7Ob3jAGT8bqJBBXFesAy7DW2G3mwhgK3Czy7j5wLvAH87yXQTUcMZVcH67Ks77stgD5DDsur/UGV4O+A445/ytJZX9ArAZSAAuOetCNaAK8K3LMnjMZfoJwK/Ybeg88EQa3zsAuy+7E4gGyqazjFYBo9IZ/woQ6vIb9koRzy9OPBeAPdjEOB44ht3eBmZz2ZYGPnKW7XHnc0XTiHEC8JPL+1pOWa77oR7ARme5bQICXbblxSnKexH40iXmJ1zG3QkEO+UsBxo7wx8DZrpMFw5MdV4Xcab3TXd9zcTKHA94ZzDds/y9I9sOTHQZ1w0noTlBDcDu/K5PpZy52LOG9OaVUUKLB94AigMlgeeA2S7T3wbsdl7Xxp793erE1tN5XzWV+RYFQoCngGLOjxuZ9D1II1GlUs5V0wHNnbK6OWU/i00Y3i47ri3OSlbVWameSbnD4e+E9DpQyvn+ndKI46qE5gyfAqxIpewW2J1DdUCABkD9tMrCJosDwPVOXN5cndDigYed8fdgk3U5l++cVkJrAsSnmF/yRoXdqV3AbjjewCjsQVR5l9j2YM+mSmN3zC+ksZzKAheBti7DtuAkCieWy8Cbzu9XOqN1II35pJXQop310gu7Q1qe4jMbnfWipDPNduwBXTGgMXAY6OpM/yR2R1gLKIHddqZnYh8QkmJYaewOcpzz2/XFrr8+zvj52O2onRPXdJydJVfvdGcDPzu/mTfOAaXzHb7CbsfeQCBQIo0Yt+JygAz8AMzCbgNNgCPAIGfcBGe9u8dZXiXTKPNt4Dfsun4IuC+dZfQWdt8wFmiayvjh2O3Gy5nmPM6BhBNPHPaA1hv4AJv83nB+wyHYddc7G8s2yFmG5YCKwDJgchrfYYJLOUWx+7konIMIoJEz3278vR8/gd0+KmMPKGq4lLcX6OMS8xMu++gwoKWzPB7FJjcvoDVwxJnOD7v/2OO8bw0cynA7ymBlHgmcyMTGuA+Y4Lz+F/CXy7hu2LO389gd1lZgWBrl/A68nsG8Mkposbis+EBD7MZWymUDes5lo5mZovxFwL2pzPdG5wcs4jJsDs6OEPcS2qs4RzPOey/sStzBZcc1ymX8QGBHyh0O0B046hpjOnGkldAmANtTKbsZdifWnRQHOKmVhU0aT6UyzDWhHUwxfhtObQDuJbSxwMoU47ckrXdOHK5HjBOB+eksq1nA+87r5tgzzoou4ztjzyZOYzfsz0hjR5nOPNJKaK5Hza2B8yk+M8LlfVdgX4oyXgQ+cV4fBG5wGVcfmzAlnbhSS2i3Y6vXXYf9wt/7gPnApy7jko72y+Gy08UmnESgXirznYitqbgqQaQybXJCw+5gDVDLZfzkpN/XWb+3ZVCeNy7bHDa5rEhn+mLA49jaiFjsQcSgdKYP5e+DjAnARpdxN2LPOEs674tgE55vFpdtSWziruIy7S3AljRiSkqs553PRQK3uoz/N/BBis+sB/o5r38CHnVet8OeXbkm4aSENgd4PJV1vwV/n4U1wG7PU7AHnjWxZ29fpha7619G1zXOAFXSq2cWkRuwG8ZcZ9BXgL+ItHSZ7JgxpoIxppIxpqUxZu5VBf09v5oZxJSRU8aYmKQ3xpgQ7FnL7SJSCrjDiRFsVekQETmf9IfdOaUWQy3s0YPrtZFD2LM8d9VyykqKOQGbmFzLPpJivrVSKacONkmke/0mA7WxBx5XMMbswB7hvwqEi8hs51pSeo5kMD4sxfu0vldWXbE8Xcp2XZ4nXF5HA2XSKW8GMFxEks4kfzDGnEsaaYxZZYwZZOz1pR7Yayj/dCN+VxnF6bqM6wG+KdbniUANERHs+rHQZdwW7E4kq3ccZmb5JsdljDmG3Umm3K58gARjTMqyAD4BNgALnBvEXsrkddha2IOdY5mJLQ29sZdb/ue8nw3cKCL1U5vYGBNrjHnbGBOIPRP6BJgjInUg+Y7eYJflXgebdJKcdHl9CYg0xlxyyk7E1gC4/u6ZWbZ1sAfGIS7znYs9E07LImNMBSe2P4COLuPqAfenWLea8/f2+hX2TBRgBPY+gavuJ3DKeSlFOeWA2s53XYU9MekCrHD+umAP1lakEzuQ8U0ha7FHo/3TmeZe7Gn5VhE5gc3aYDf8rFoM9BaR0ulME409sktSI8V4k8pn5mAXdj/s9ZAQZ/gR7BlaBZe/0saY11Mp4xhQJ8VGVRebeNx1DPtDAyAiXtgN0LXsOinm67rBJjmC3aFl6wYM58ClL3ZlvooxZoYxphP2CKoE9toApL7M0xueJOWdgK7f6yJp/84ZlXvF8nQpO7u/1RLsdnAbdmP9Mq0JjTFrsdewmmdzXlnluiyOYKvTXdfnssaYAcYeCh8FeqQYX8IYk9U7FzOzfJPXVxGpiT3rOZ7iM2GAl4ikLAtjzCVjzFPGmMbATdj9yYBMxuYtIq4HRiljy2j9udeJd6+zT0uqerw7o5kbYy5iz6wBGouIP7Y6ehRQyUkYR5zysiuzyzYBqOPyW5c3xmR4AG6MOQ88ADwmIg2dwUeAD1PZV37ijF+APZFpiK3q/+rqkpPL+WeKckoZYxY641dgE1onbHJbga0VuhF3E5oxJgJ7DeojEenv3J5aVERuEZEpIlLCCf4BbJ1o0t8jwMhs3Hk10/nC3zm3ZhcRkcoi8pSI3OpMsxUYISJeItIHm7kzMhfoBTzElQt6FvbMrbdTXgmxjxmkdsv1euxO9p/OMuiGrXpJ62wzK74GBohIF+cs4Ens2eoml2keFZGaYh9feNL5TEqrsFUFLzu/VUkR6ZTRzJ3v0wx7Z1NZ4KpbbEXET0S6ikhx7FHkJewGA/YIs75zFpAVdZyjV28RuQu74/nNGbcVe1bkLSIduPKu2nDsjrBuGuX+ALQSezu9t4jc45T9axbjA5KPkmdh7/Yt6lqOiHQXkfuS7gx1luNt2GrNvLbKiWGCsy57i0iAiLR2xn8KvO5y5lBNRLLzqMNSoJyIPOTM41bsDsf1zszBItLG2Ue8AvxijLngWogxJhp7sPmBiFR1yurqxNYraR+AvR6awN/rW5qMMZHYa3KvO9tAY+y1vlmZ+WIiUhFbizOCK/dpT5HGQbqITBaRTs4yL4a9oSMOez2zrBP3KaCIiPwfVx/IZVVml+1s4D0RqSRWPRG5KTMzMMaEYfdtTzuDpgN3OfuAIs6+pVfSeu8k8h+xZ6eXnQO71PwXmCj22UERkbIiMkD+vptyBfYE6oyTf5Zjf4sYlxORNGV4JG+MeQdbbfEM9kc5gv3B5jszvoSt2zyR9Ad8jj3d7ZNR+SnmdRl7O/Ru7PW0C9hqhyr8feb3f9hEch57jW9+Jso9jj3b7IRLIjDGHMHuKJ9y+W6TSGW5GGNisSv6LdhrJR8D9xhjdmflO6YR3zZgDPbHPoU9Iu2X4pR9Lvai7j7shjIllXLisDe4tMAeoR3GXm9Ly70iEoldlt9jj2LbGWPCU5m2JPZC+Wns0WAZ7MFOUmylgLMisiYTXznJSuzdcmexG84AZyUG+5v4O7H9C5cDB6e6bwqw2am2cK3exhhzEvtbPY09MBiPvdP1fBZiS2kG9qzkqxS/yznsM4g7RCQKu1HPxia/pGe6Nrsx30xz+f07YavZTmF3MEnVVVOwtSBLnd99Dfa6XFbncxGbtO/BLt9/Y68ZuVblzcQug1PYg4n70yjuQez6tA27biWtU/WwiSkSe3DzLfYsIDPuw66PR7AHSO8bY+Zl8rNDnc/NS7FP+wSoKSmev3XEOuPDsQd3w7B3QIdj9ztfYu+gPordlwVnMpa0ZHbZPoy9C3WL8/8n7F3QmfUW9uShvrOfG4b9rc9grwM+nGL6r7D77zlpFWiMWYqtjv8Cu23vxt74knTW/Cf27HWlM/1R7HJdmZmAxbkop/Ixp9pjsDFmladjUSojIjIfWGWMecvTsRQ2umzTV+AedlVKKaVSowlNKaVUoaBVjkoppQoFPUNTSilVKORl4725rkqVKsbX19fTYSilVIGyefPm08aYHG2U3BMKVULz9fVl06ZNGU+olFIqmYik1lpLgaNVjkoppQoFTWhKKaUKBU1oSimlCoVCdQ0tNXFxcYSFhRETE5PxxKpQKFGiBD4+PhQtWtTToSil8lChT2hhYWGULVsWX19fst52ripojDGcOXOGsLAw6tdPtbcPpVQh5ZEqRxH5QkTCRSTVRjqdVpjfF5EQEdnm0lp4lsXExFC5cmVNZtcIEaFy5cp6Rq7UNchT19CCSL8l/luwXX43wnZN80k602ZIk9m1RX9vpa5NHkloxpiVpNIrsot+2C5pjDFmHVBBbEd2SimlXOzavpk3f3G3R5rCIb/e5VibK7tJD+PKLtSTicgDIrJJRDadOnUqT4LLKi8vL1q2bEnz5s25/fbbOX8++91y+fr6cvr01R0M+/r6cuONN14xLGmeWdGtW7cMH05Pa5q4uDiefPJJGjVqRPPmzWnfvj2//PJLluavlMqYMYZ1e4/z3TvjuW5eT8z6aRyPuOTpsDwuvya01OqMUm1F2Rgz1RjT1hjTtmrV/NlyS8mSJdm6dSvBwcFUqlSJjz76KFfmExkZyZEj9jhg165duTKP9Dz77LMcP36c4OBggoOD+fHHH4mMjMz05+Pj4zOeSKlrmDGGZbvDmfxBEBVm9WTQhZkcqtGLh//vX9QsX9LT4Xlcfk1oYUAdl/c+wDEPxZKjOnbsyNGjR5Pfv/nmm7Rr146AgACef/755OH9+/enTZs2NGvWjKlTp2aq7DvvvJOvv7Ydcs+ZM4fhw4cnj4uJiWH06NH4+/vTqlUrli1bBsClS5cYNmwYAQEBDB06lEuX/j7K++233+jYsSOtW7dmyJAhREVFpTnv6Ohopk2bxgcffEDx4sUBqF69OnfeeScAZcqUSZ523rx5jBo1CoBRo0YxceJEunfvzqRJk/D19b3iDLZhw4acPHmSU6dOMWjQINq1a0e7du1YvXp1ppaJUoVBYqJh4fbj9P1gFR/MmM2/z06kTsnLxN45h0YPzaVMxRqeDjFfyK+37f8AjBeRuUAgEGGMOe5uoS/+uIOdxy64HZwrv1rleP72ZpmaNiEhgSVLljBmzBjAJox9+/axYcMGjDHccccdrFy5ki5duvDFF19QqVIlLl26RLt27Rg0aBCVK1dOt/zBgwczatQonnjiCX788Udmz57NzJkzAZLPCrdv387u3bvp1asXe/fu5ZNPPqFUqVJs27aNbdu20bq1vaH09OnTvPLKKyxevJjSpUvzxhtv8M477/Dcc8+lOu+QkBDq1q1LuXLlMrUsXO3du5fFixfj5eVFYmIi33//PaNHj2b9+vX4+vpSvXp1RowYwWOPPUbnzp05fPgwvXv39shZqFJ5KS4hkR+2HuPj5SGcPXWcClVqcu/AgZgYoXS70VCivKdDzFc8ktBEZA7QDagiImHA80BRAGPMp8BC4FYgBIgGRnsizpxy6dIlWrZsSWhoKG3atKFnz56ATWi//fYbrVq1AiAqKop9+/bRpUsX3n//fb7//nsAjhw5wr59+zJMaJUqVaJixYrMnTuXpk2bUqpUqeRxq1at4pFHHgGgSZMm1KtXj71797Jy5UoeffRRAAICAggICABg3bp17Ny5kxtuuAGA2NhYOnbsmINL5W9DhgzBy8sLgKFDh/LSSy8xevRo5s6dy9ChQwFYvHgxO3fuTP7MhQsXiIyMpGzZsrkSk1KedDk+gXmbw/hk+X4izp3hjXLfcnPZdXiNXY9X+ZrABE+HmC95JKEZY4ZnMN4A43J6vpk9k8ppSdfQIiIi6Nu3Lx999BGPPvooxhj+9a9/8Y9//OOK6ZcvX87ixYtZu3YtpUqVolu3bpl+rmro0KGMGzeOoKCgK4an15Frare5G2Po2bMnc+bMydR8GzZsyOHDh9NMMq7zSPldSpcunfy6Y8eOhISEcOrUKebPn88zzzwDQGJiImvXrqVkSb1OoAqv6Nh4vlp/mGl/HODkhcvcX20PT1T8lOIxp5CO46CknpGlJ79eQyuUypcvz/vvv89bb71FXFwcvXv35osvvki+NnX06FHCw8OJiIigYsWKlCpVit27d7Nu3bpMz2PAgAH885//pHfv3lcM79KlC7NnzwZsFd/hw4e5/vrrrxgeHBzMtm3bAOjQoQOrV68mJCQEsNfI9u7dm+Z8S5UqxZgxY3j00UeJjY0F4Pjx48yaNQuw19N27dqVXKWYFhFhwIABTJw4kaZNmyaflfbq1YsPP/wwebqtW7dmepkold9FXIrjw6X76PzGMl75eReNKpdgY5M5PHPhRUqUrYLcvxh6vQLFSmVc2DVME1oea9WqFS1atGDu3Ln06tWLESNG0LFjR/z9/Rk8eDCRkZH06dOH+Ph4AgICePbZZ+nQoUOmyy9btiyTJ0+mWLFiVwx/+OGHSUhIwN/fn6FDhxIUFETx4sV56KGHiIqKIiAggClTptC+fXsAqlatSlBQEMOHDycgIIAOHTqwe/fudOf9yiuvULVqVfz8/GjevDn9+/cn6c7T119/nb59+9KjRw9q1kz/kcKhQ4cya9as5OpGgPfff59NmzYREBCAn58fn376aaaXiVL51Zmoy7y5aDedX1/KW7/tpYVPeb57qCOz/tGZquXLQren4IHlULuNp0MtECS9qqiCpm3btibl81G7du2iadOmHopIeYr+7io/OxERw9SVB5iz4TAx8Qnc2rwmj7QtRZMtL0OPZ6BaUzAG8qjVGxHZbIxpmyczy0X59S5HpZQqdA6fieaTFfv5bnMYCcbQr2UtHu7agIZH5sF3z0FiPDQfaBOaNuGWZZrQlFIql+07GcnHy/fzw1/H8BJhSFsfHux6HXXMcfhxBIT+AfW7wO3vQyXtJSK7NKEppVQuCT4awYdLQ1i08wQlvL0Y3cmXsV0aUL1cCTvBktlwfBvc8QG0ulvPytykCU0ppXLYxtCzfLg0hBV7T1G2hDfjuzdk9A31qVS6GJzcAREXoU576DIJ2o2Fctr2ek7QhKaUUjnAGMMf+07z4bIQNhw8S6XSxZjU+3ru7liPciWKQvxlWPYa/PE21GwJ9y+GoiXtn8oRmtCUUsoNiYmGxbtO8tGyEP4Ki6BGuRI819eP4e3rUrKYbQGHsE2wYDyc2gUBQ6HP61q9mAv0ObQ84Nowr6tRo0Yxb968bJX5wgsv8NZbb6U6XESSH4gGePfddxGRDLuFcRUUFMT48eOzPc0vv/xC27Ztadq0KU2aNOGJJ57I9LyVKggSEg0Lth7llvf+4IGZmzkXHce/B/qz4p/duK9z/b+T2eH18NnNcPkCjPgWBk6FUpU8G3whpWdohZC/vz9z585NbjZq3rx5+Pn55dn8g4ODGT9+PD///DNNmjQhPj4+0z0GgG3EOaltR6Xym9j4RL7fYttZDD0TTaNqZfjP0Jb0DaiJt5fLOcLF01C6Cvi0g14vQ+t7oUTWG+9WmadnaHnIGMP48ePx8/PjtttuIzw8PHnc5s2b6dq1K23atKF3794cP247F5g2bRrt2rWjRYsWDBo0iOjo6Azn079/fxYsWADAgQMHKF++PK59xc2ZMwd/f3+aN2/O5MmTk4dPnz6dxo0b07Vr1yu6Z8lq1y1Tpkzh6aefpkmTJgB4e3vz8MMPA1eflSadvS5fvpzu3bszYsQI/P39mTx5Mh9//HHydC+88AJvv/02kHaXO0rlppi4BIJWH6Tbm8uY/N12ypTw5tO7WrNoQhf6t6r9dzK7dB5+eAQ+aAORJ6BIEej0iCazPHDtnaFNv+3qYc36Q/uxEBsNs4dcPb7lCGg1Ei6egW/uuXLc6J8zPevvv/+ePXv2sH37dk6ePImfnx/33XcfcXFxPPLIIyxYsICqVavy9ddf8/TTT/PFF18wcOBAxo4dC8AzzzzD559/ntxqflrKlStHnTp1CA4OZsGCBQwdOpTp06cDcOzYMSZPnszmzZupWLEivXr1Yv78+QQGBvL888+zefNmypcvT/fu3ZN7Afi///u/LHXdEhwczOOPP57p5ZJkw4YNBAcHU79+fbZs2cKECROSE+E333zDr7/+mm6XO0rlhsiYOGatO8znqw5wOiqWdr4VeW2gP10bV726Ye/dP8NPE+HiKSeJaWPCeenaS2getHLlSoYPH46Xlxe1atWiR48eAOzZs4fg4ODkbmUSEhKS2zsMDg7mmWee4fz580RFRV3V6HBahg0bxty5c1m0aBFLlixJTmgbN26kW7duyWdsI0eOZOXKlQBXDB86dGhyY8Rpdd2S09q3b0/9+vah0latWhEeHs6xY8c4deoUFStWpG7durz//vtpdrmjVE46dzGW6WtCCVp9kAsx8dzYqArjuzcksEEq3TglxMH/xsKO76G6P4yYC7Va5X3Q17hrL6Gld0ZVrFT640tXztIZWWrS6qqlWbNmrF279qpxo0aNYv78+bRo0YKgoCCWL1+eqfncfvvtTJo0ibZt217R6WZWu5GBrHfd0qxZMzZv3kyLFi2uGuft7U1iYmJyLEkt88OV3ciA7bB03rx5nDhxgmHDhiV/JrUud5TKKeGRMXz2x0FmrTtEdGwCvfyqM657Q1rUqZD2h7yKQrHSth3GGybY9yrP6TW0PNSlSxfmzp1LQkICx48fZ9myZQBcf/31nDp1KjmhxcXFsWPHDgAiIyOpWbMmcXFxyd28ZEbJkiV54403ePrpp68YHhgYyIoVKzh9+jQJCQnMmTOHrl27EhgYyPLlyzlz5gxxcXF8++23yZ/JatctkyZN4rXXXks+w0tMTOSdd94BwNfXl82bNwOwYMEC4uLi0iwn6Sxz3rx5DB48GCDNLneUclfYuWienR9M5zeW8dkfB+jpV51FE7ow9Z62qSez80dgznA46dRe3PGhfVBak5nHXHtnaB40YMAAli5dir+/f/LNFwDFihVj3rx5PProo0RERBAfH8+ECRNo1qwZL7/8MoGBgdSrVw9/f/8sVfUlndW4qlmzJv/+97/p3r07xhhuvfVW+vXrB9gbLzp27EjNmjVp3bo1CQkJgO26Zdy4cQQEBBAfH0+XLl3S7b4lICCA//znPwwfPpzo6GhEhNtus9cux44dS79+/Wjfvj033XTTVWdlrpo1a0ZkZCS1a9dOroLt1asXu3btSu49u0yZMsyaNYtq1aplerko5erAqSg+Xr6f+VuOIgKDWtt2Fn2rpLFuJibC5i/g9+fBJELAnVDdT58rywe0+xhVKOnvrjKy89gFPloewsLtxynmVYTh7evyQJcG1KqQTtX66RB7B+PhNdCgG9z+HlT0zaOIc492H6OUUgXQn4fP8dHSEJbsDqdMcW8e7HodYzrXp0qZ4hl/+K85EL4D+n0ELUfqWVk+owlNKVXoGWNYe+AMHy4NYc3+M1QoVZSJPRtzb0dfypfK4JrXie0QexHqdrDXyNqPhbI18iZwlSXXREIzxqR5B58qfApTNbpyjzGGZXvC+XBpCH8ePk/VssV5+tamjAisS+niGez+4i/Dyjdh1bsujQmXgKKazPKrQp/QSpSNLy2UAAAgAElEQVQowZkzZ6hcubImtWuAMYYzZ85QokQJT4eiPCgh0fBr8Ak+WhbCzuMXqF2hJC/3a8aQtnUoUTQTzaodXg8/jIfTe6HFcOj9mlYvFgCFPqH5+PgQFhbGqVOnPB2KyiMlSpTAx8fH02EoD4hLSGTB1mN8vDyEA6cu0qBKad4cHED/VrUp6pXJp5QOr4Mv+kB5Hxj5HTS6OXeDVjmm0Ce0okWLJrc+oZQqnGLiEvh2cxj/XbGfsHOXaFqzHB+OaMUtzWviVSSTZ1ZR4VCmGvi0h96vQut7oHjZ3A1c5ahCn9CUUoXXxcvxfLX+MNP+OEB45GVa1a3Ai3c0o0eTapm/xHDpHCx6Bnb/CA+vt71HdxyXu4GrXKEJTSlV4ERcimPGmlCmrz7Iueg4Ol1Xmf8MbUnH67J4rXznD7DwCdvVyw3/ByUr5l7QKtdpQlNKFRinoy7z+aqDzFx7iKjL8dzUpBrjejSkdd0sJqKEOJh3H+z6AWr4w8hvoebVbY+qgkUTmlIq3zsecYn/rjjA3I2HuRyfyK3+NRnXrSF+tbLZx5hXUdu1y03PQadHtf3FQkITmlIq3zp05iKfLN/Pd3+GYQz0b1Wbh7pdx3VVy2S9sPOHYeEkm8SqN4N+H2b8GVWgaEJTSuU7e09G8tGyEH786xjeXkUY2q4O/+hyHXUqlcp6YYmJsPEzWPyCfX96n01oqtDRhKaUyje2h0Xw4bJ9LNpxklLFvBjTuT5jb2xAtXLZfFD+9D5YMB6OrIPrboLb/wMV6uZs0Crf8FhCE5E+wHuAF/CZMeb1FOPrAjOACs40TxpjFuZ5oEqpXLfh4Fk+XBbCyr2nKFfCm0d7NGT0DfWpWLqYewX/NRdO7Yb+n9gWP7S1j0LNI93HiIgXsBfoCYQBG4HhxpidLtNMBbYYYz4RET9goTHGN71yU+s+RimVPxljWLnvNB8tDWFD6Fkqly7GmBvrc3eHepQt4cZNGsf/gthoqNcR4mIgJgLKVs+5wAsh7T7GPe2BEGPMAQARmQv0A3a6TGOApFuYygPH8jRCpVSuSEw0/LbzJB8tC2H70Qhqli/B87f7MaxdXUoWy0Q7i2mJi4EVr8Pq96F2axjzu9OYsLbrea3wVEKrDRxxeR8GBKaY5gXgNxF5BCgNpNqgmog8ADwAULeu1o0rlV/FJyTy07bjfLw8hL0no6hXuRSvD/RnYGsfinlnsp3FtBxaaxsTPhMCre6CXq9o9eI1yFMJLbU1LWXd53AgyBjztoh0BGaKSHNjTOIVHzJmKjAVbJVjrkSrlMq2y/EJ/O/Po3y6Yj+HzkTTuHoZ3hvWktv8a+Kd2QaD03N4HUy/BSrUgbu/h+t6uF+mKpA8ldDCgDou7324ukpxDNAHwBizVkRKAFWA8DyJUCnllkuxCczZcJipKw9w4kIMAT7l+e/dbejZtDpFMttgcHoiT9prYz7toc/r9syseDaeT1OFhqcS2kagkYjUB44Cw4ARKaY5DNwEBIlIU6AEoH3AKJXPRcbE8eXaQ3yx6iBnLsbSvn4lpgwO4MZGVXKmT8Los7DoKdi9EMY5jQl3eND9clWB55GEZoyJF5HxwCLsLflfGGN2iMhLwCZjzA/A48A0EXkMWx05ymhXxErlW2cvxjJ99UGC1oQSGRNP18ZVGde9Ie3rV8q5meyYbxsTvnQOOj8GpXKwbFXgeew5NOeZsoUphj3n8noncENex6WUyprwCzFM++MAs9cfJjo2gd7NqjO+eyP8fcrn3EwS4mDeaNj1I9Rsaa+V1fDPufJVoaAthSilsuXI2Wj+u3I/32wKIz4hkTta1OLh7g1pXD0XOsX0KgqlKsPNL0LH8eCluy51NV0rlFJZEhIexSfL9zN/61GKCAxu48ODXa+jXuXSOTujc6Hw8xNw8wtQoznc/l7Olq8KHU1oSqkMGWPYGHqO6asP8uuOExT3LsI9HevxQJcG1CxfMmdnlpgAG6bCkpdAisDZAzahKZUBTWhKqTTFxCXww9ZjBK0JZefxC5Qr4c1DXa/jvs71qVKmeM7PMHw3/PAIhG2Ahj1tY8LlfXJ+PqpQyrGEJiL1gFLGmF05VaZSyjOOnr/ErHWHmLvhMOei47i+elleG+BP/1a1KFUsF4+Dg+fZ1j4GTgP/Idrah8qSHFkzRWQy0BZIFJFLxphROVGuUirvGGPYcPAsQWtCWbTjBAA9/apzbydfOjaonDPPkKXm2BaIuwT1OsGNT0D7f0CZqrkzL1WoZSuhichDwH9dmqFqbYwZ4ozbllPBKaVy36XYBBZsPUrQmlB2n4ikQqmijO3SgLs71MOnYjY61MysuEuw/N+w5kOo3QbG/KaNCSu3ZPcM7RLwq4i8a4z5BVgiIkuxbTQuybHolFK5JuxcNDPXHeLrjUc4Hx1HkxpleX2gP/1a1nav1fvMCF1tr5Wd3Q+t74GeL2v1onJbthKaMSZIRL4BJjut3T8LzAGKGWPO5GSASqmcY4xh3YGzBK05yO87TwLQu1kN7u3kS2D9SrlXrejq0FoIuhUq1IN7FkCDbrk/T3VNcOcaWh1sj9KXgVeAGOD5nAhKKZWzLsUmMH/rUWa4VCv+o+t13NWhHrUr5PBt92m5cNy2u1gnEG55E1qNhGI5/OyauqZl9xra59g+ykoCO40xo0WkLTBdRFYZY/6dk0EqpbLnyNloe7fixiNEXIqjac1yTBkUwB0ta1GiaC5XKya5eAZ+fRL2LoJx66BcLQh8IG/mra4p2T1Da2uMaQEgIluAfxljNgG3icigHItOKZVlxhjW7j9D0JpQFu86iYjQu1l1RnWqTzvfinlTrWgDgR3/g4X/hJgIuPFx23yVUrkkuwntd+cmkGLA164jjDHfuR2VUirLomPj+X6LrVbcezKKiqWK8qBTrVgrr6oVkyTEwTf3wJ6FUKs19PsQqjfL2xjUNSe7N4U8ISKVgARjTEQOx6SUyoIjZ6P5cm0oX288woWYeJrVKseUwQHc0SIPqxVT8ioKZWtAr1egw8NQxENxqGtKtm8KMcaczclAlFKZZ4xhzf4zTF8dypLdJykiQp/mNRjVyZe29fKwWtHV2YPw8+PQ8yXb9mLfd/M+BnVN07YclSpAomPj+d+ftlpxX3gUlUoXY1y3hozsUDfnGwnOrMQEWP8pLHnZnpmdC9XGhJVHaEJTqgA4fMapVtx0hMiYeJrXLsdbQ1rQN6Cm56oVAU7uhB/Gw9HN0LgP3PYOlK/tuXjUNc3thCYi7QFf17KMMV+5W65S1zpjDKtCTjNjTShLdofj5VQrjr7Bl9Z1PVStmNKO/9kzskGfQ/NB2tqH8ii3EpqIBAF+wFYgwRlsAE1oSmXTxcvx/O/PMGasPURIeBSVSxdjfPeGjAysR43y+aCdw7DNEB8DvjdAl0kQ+CCUruLpqJRy+wytA+Dn0kixUiqbQk9f5Mu1h/h20xEiL8cT4FOed+5swW0BNSnunQ/uEoyNhmWvwrqPwacd3LcIvIvbP6XyAXcT2g6gChCeA7Eodc1JTLTVikFrQlm2x1Yr3upfk1E3+NKqToX8Ua0IcHClbUz4XCi0GQ09X9TqRZXvuJvQygO7RGQdtk1HAIwxA90sV6lCLepyPN9tDmPG2lAOnLpIlTLFeaRHI0YG1qV6uXxQrejq0BqYcTtUrA/3/gT1b/R0REqlyt2Epm02KpUFB09fZMaaUOZtDiPqcjwtfMrz7tAW3OqfT6oVXV04ZttdrNsRbnsbWoyAYrnYP5pSbnIroRljlohIFWxv1QCbjDGn3Q9LqcIjMdGwct8pgtaEsnzPKYp6Cbf51+TeTr60qlvR0+Fd7eJp+GUy7PsNxq23Sa3d/Z6OSqkMuXuX4yDgXeAPbOeen4rIY8aY73MiOKUKssiYOKda8RAHT1+katniTLi5ESMC61KtbD6rVgTbmPD2efDLP+FypL2DsZTevagKDnerHJ8D2hljTgKISHXgN0ATmrpmHTgVxZdrDyVXK7asU4H3hrXkluY1KeZdxNPhpS4hDr6+C/b+CrXb2saEqzX1dFRKZYm7Ca1IUjJznALy6RarVO5JTDSs2GurFVfstdWKfQNqcW8nX1rWqeDp8DLmVRTK1Yber9nnyrQxYVUAuZvQfhORhfz9IPUwYJGbZSpVYFyIiWPepjC+XBtK6JloqpUtzmM3N2Z4YJ38Wa3o6sx++Hki9HwZagZA33c8HZFSbnE3oT0B3AncgL2GNgOY525QSuV3IeFRfLk2lO82h3ExNoHWdSvwWM/G+btaMUlCvH04etmr4FUcIo7YhKZUAefuXY4G28Hn1xlNq1RBl5hoWL43nOmrQ/lj32mKeRWhb4uajOrkS4BPAahWBDgRbBsTPrYFrr/N3o5frqano1IqR2QroYnICmNMVxE5h227MXkUNs9VyuDzfYD3AC/gM2PM66lMcyfwglP+X8aYEdmJVSl3XYiJ41unWvHQmWiqlyvO4z0bMzywLlXKFLBmn3b9ABFhMCQI/Pprax+qUBF7kpXFD4kUMcYkikiqV46NMQmpDXc+6wXsBXoCYcBGYLgxZqfLNI2Ab4AexphzIlLNGJNh81pt27Y1mzZtyuK3USp1IeGRzFhziO/+DCM6NoG29Spybydf+jSvQVGvfF6t6Cpsk9OYcGeIvwyxF6FUusec6hojIpuNMW0znjJ/y9YZmktjxJ8bY0a5jnNa4B9F2toDIcaYA870c4F+wE6XacYCHxljzjnz07YiVZ5ISDQs2x3OjLV/Vyve0bIWozr50rx2eU+HlzWxF2Gp05hwnUC471dtTFgVau7eFHLFlWTn7KtdBp+pDRxxeR8GBKaYprFT3mpsteQLxphfUytMRB4AHgCoW7dupgNXylXEpTi+3XSEL9ce4vDZaGqUK8Gk3tczrF0dKhe0akWAA8vhh0fh/CHbysdNz2v1oir0snsNbTLwJFBWRM4mDcZe7/o8o4+nMixlvac30AjoBvgAf4hIc2PM+as+aMxUYCrYKsfMfgelAPadjCRoTSj/+/Mol+ISaOdbkcl9mtCrWfWCVa3oKnQ1fNkPKl0HoxbafsuUugZk9wxtCvA2tnHiJ5MGpnftzEUYUMflvQ9wLJVp1hlj4oCDIrIHm+A2ZjNepZIlJBqW7DrJjLWhrA45QzHvIvRrYR+CLnDViq4iwqC8D9TrBH3fhRbDoWhJT0elVJ7J7jU0A8SLyPe4VBcm9d1kjFmTzsc3Ao1EpD5wFPswdso7GOcDw4Egp/HjxsCB7MSqVJKI6Di+2XSEGWtDCTt3iZrlbbXi8PZ1qVS6mKfDy76ocFg4CUKWwPgNtjHhtvd5Oiql8py719CedXldAmgDbAG6pvUBY0y8iIzHtijiBXxhjNkhIi9hW+v/wRnXS0R2AgnAJGPMGTdjVdeoPSdsteL8LbZasX39Sjx1a1N6+VXHu6BWK4JtTHjb1/Drk/YGkK6ToXRVT0ellMdk67b9NAsT8QVe89QzY3rbvkqSkGhYvOskM9aEsmb/GYp7F6F/y9rc28kXv1rlPB2e++JjYe4ICPkdfNrbxoSrXu/pqFQBdU3ftp8WY0yoiDTPyTKVyorz0bF8vfEIM9cdIuzcJWpXKMnkPk0Y1q4OFQtytWJK3sWgUn3o8wa0H6uNCSuF+/2hvcvfdygWAVoBO9wNSqms2n3iAjPWhPL9lqPExCXSoUElnrmtKTc3LeDViq5Oh8DPj0GvV23bi7e+6emIlMpX3D1DC3Z5HQ98b4xZ4WaZSmVKfEIii3edJGhNKOsOnKVE0b+rFZvWLATVikkS4mHtB7Ds31C0hL2bURsTVuoq7jZO/LmIFMXeUm+AfTkSlVLpOHcxlrkbjzBr3SGOnrfViv+6pQlD29WhQqlCVK0IcGI7LBgHx/+CJn1tY8Jla3g6KqXyJXerHHsD04DD2AemfURkrDHmt5wITilXO4/ZasX5W49yOT6Rjg0q82xfP25uWq3wVCumtOsnuHAc7vwS/Pp5Ohql8jV3qxzfA242xuwFEJHGwAJA+25XOSI+IZHfd55k+ppQNhy01YoDW/twb6d6NKlRiKoVXR1eBwmxUL8L3Pg4BP5DGxNWKhPcTWjhSckMwBizV0ROuVmmUsTEJfDF6oPMWnuIYxEx+FQsyVO3NuHOtoWwWjHJ5ShY8hJsmAp1O9iE5l0MvDWZKZUZ2W3L8Q7nZbCI/IDt6sUAQ4ANORSbukbFxCVw/4xNrAo5zQ0NK/PCHc24qWl1vIoU4sZ1Q5bAjxNs79HtH4CbnvN0REoVONk9Qxvi8joC6O28jgSquRWRuqbFxify8Ow/WRVymreGtGBwGx9Ph5T7QlfDrIFQuZHt4qVuB09HpFSBlN22HO/O6UCUik9IZMLXW1i6O5xXBzQv/Mns/GGoUNc2Jnz7+xAw1N6Wr5TKluxWOT5ujHk7xYPVyYwxE92OTF1TEhMNk+ZtY+H2Ezzb14+RgfU8HVLuiTwJC5+A/ctg3HooXxva3OvpqJQq8LJb5bjf+R+c7lRKZYIxhqfnb+f7LUeZ1Pt6xnSu7+mQcocxsPUrWPQUxF2CbpOhjNbQK5VTslvlON/pnbqRMebJDD+gVBqMMbz4407mbDjC+O4NGde9oadDyh3xsTBnGOxfAnU6wB0fQNXGno5KqUIl27ftG2MSRKR9Tgajri3GGKYs2kPQmlDu71yfx3sV4h28dzGo3BCuvwXajoEihfRBcKU8yN3n0P4Ukf8B3wIXkwY6fZopla4PlobwyfL9jAysy9O3NU3uILbQOLUXfnoMer8KtVrCrVM8HZFShZq7Ca06NpHd6jLMAJrQVLqmrtzPO7/vZVBrH17u17xwJbOEOFj9Hqx4A4qVhsgTno5IqWuCuwntI2PMOtcBIqIP0ah0zVwbymsLd9M3oCZTBgdQpDA9MH1sK/ww3jYq7NffdvGiN34olSfcrcj/OJVhH7lZpirEvtl0hGcX7KCnX3XeHdqy8LX+sfdXiAqHobPgzhmazJTKQ9l9Dq090BGoKiKPuowqBxTNicBU4bNg61Emf7eNGxtV4cMRrShaWFrIP7TWNibcoCt0nmgbEy5Z0dNRKXXNyW6VY2mgivP5qi7DI7myWSylAPg1+AQTv/mLwPqVmHp3W4p7e3k6JPddjoTFL8LGaVC3k01o3sXsn1Iqz2X3ObRlwDIRmW6MOQAg9qp+KWPMxfQ/ra41y/aE88icP2nhU57P7m1HyWKFIJntWww/TbC9Rwc+BD2e8XRESl3z3K3zeUFEyolIKWAHcFBEtNkrlWxNyGkenLmZ62uUZfro9pQp7u59SPlA6CqYPQiKloIxv8Etr0PxMp6OSqlrnrsJzd8YcwHoD/wG+ACj3A1KFQ6bQs9y/5eb8K1cmpn3BVK+ZAG+vGoMnAu1r+vdAHd8CA/+AXW0bQGl8gt3E1oxEfEG+gHzjTGxQKL7YamCblvYeUZP30iNciWYeX97KpYuwNeVLhyHr++CT26AiKMgAq3vBu/ino5MKeXC3fqfz4DD2EaKV4hIXSDK7ahUgbbr+AXu/nwDFUoXZfbYQKqVLaBdohgDW2bComcg4TJ0fwrKVPd0VEqpNLiV0Iwx7wLvJr0XkSNAD3eDUgVXSHgUd3++npJFvfjq/g7ULF/S0yFlT3wsfDUEDix3qhg/gMrXeToqpVQ6svsc2nBjzJwUz6C5et+NmFQBdejMRUZ+tg4QvhobSJ1KpTwdUtYZY6sUvYtBNT9oege0Ga2NCStVAGT3DC3pqdGq6U6lrhlHz19ixLT1xMYnMveBjjSoWgDv+gvfbW/F7/O6bUy4z789HZFSKguy+xzax87/Z3M2HFUQhV+IYeS0dVyIiWPO2A5cX6Osp0PKmoQ4WPUfWDkFipWxTVcppQqcbF9DE5EbgfFAE2fQLuBDY8yqnAhMFQxnoi4z8rP1nIq8zMz7A2leu7ynQ8qaY1tgwXg4GQzNB0GfN6CMVjwoVRBl9xraLcAnwKvAFECA1sCXIvKQMWZRzoWo8quI6Dju/nwDh89GM+O+9rSuWwDbL9z7G0SfgWFzoMmtGU+vlMq3snul+5/AAGPMNGPMZmPMJmPMVGAg8K/MFCAifURkj4iEiMiT6Uw3WESMiLTNZqwqF0TGxHHP9A2EhEcx9Z62dGhQ2dMhZV7oKti/zL7u/Bg8vE6TmVKFQHYTWi1jzJaUA40xW4EaGX1YRLyw3czcAvgBw0XEL5XpygKPAuuzGafKBZdiExgTtIkdRyP4aGRrujYuIFV0MRdsD9JBt8HKN+0w72JQsoJn41JK5YjsJrT0Hp7OTOPE7YEQY8wBp3WRudjWRlJ6GVulGZP1EFVuiIlLYOyXm9h06Cz/GdaSnn4F5EHjvYvg4w6wOQg6joeR8zwdkVIqh2X3ppDrROR/qQwXoEEmPl8bOOLyPgwIvKIgkVZAHWPMTyLyRFoFicgDwAMAdevWzcSsVXbFxify8Ow/WRVymreHtKBvQC1Ph5Q5oavgqzuhalO480vw0dprpQqj7Ca0QemM+zATn0+tm2KTPFKkCLYFklEZFeRcu5sK0LZtW5PB5Cqb4hMSmfD1FpbuDufVAc0Z1MbH0yGlL6kx4Ur1bUsf/T+B5oO1rzKlCrHsPoe2xM35hgF1XN77AMdc3pcFmgPLbTdr1AB+EJE7jDGb3Jy3yqLERMOkedtYuP0Ez/b1Y2RgPU+HlL4Lx+Dnx+HgShi3AcrXhpYjPB2VUiqXeapzqo1AIxGpDxwFhgHJexxjTAS2R2wARGQ58IQms7xnjOHp+dv5fstRJvW+njGd63s6pLQZA3/OgN+etQ9L93gaymZ4j5JSqpDwSEIzxsSLyHhgEeAFfGGM2SEiLwGbjDE/eCIudSVjDC/+uJM5G44wvntDxnVv6OmQ0hYfazvdPLgSfG+E29/TxoSVusZ4rPtgY8xCYGGKYc+lMW23vIhJ/c0Yw5RFewhaE8qYzvV5vFdjT4eUOtfGhKv7Q7OB0PpebUxYqWuQW1u9iPwqIhVc3lcUkZ/dD0t52gdLQ/hk+X5GBtblmdua4lzLzF9O7oTPe8HRP+37Pq9BW20ZX6lrlbtnaNWNMeeT3hhjzolIAbmXW6Vl6sr9vPP7Xga19uHlfs3zXzKLj4U/3rZ/JcrZpquUUtc8dxNaooj4GGPCAJweq1UBNnNtKK8t3E3fgJpMGRxAkSL5LJmFbYYF4+DULvC/03b1UroANbullMo17ia054DVIrLUed8deMjNMpWHfLPpCM8u2EFPv+q8O7QlXvktmQHsXwKXL8CIb6Bxb09Ho5TKR8QY955FFpHqQEfsw9KrjTEe60yqbdu2ZtMmvbM/OxZsPcqEr7fSuWEVPru3LcW9vTwd0t8OroTEeLiuh61ujI+xVY1KqRwhIpuNMQW+CZ3sdh/TyBizT0QCnEEHnP81RKSGMWZbzoSn8sKvwSeY+M1ftPetxNS781Eyu3Qefn/OPltWr7NNaN7FtLUPpVSqslvl+CQwBttifkoG6JLtiFSeWrYnnEfm/EmAT3k+H9WOksXySTLb84ttGT/qJHR6BLo95emIlFL5XHabvhrj/L8xZ8NReWlNyGkenLmZ62uUJWh0e8oU99hjiVc6+AfMGQbV/GDYbKjdxtMRKaUKALf2YCJSHPgH0Bl7ZvYHMM0YczkHYlO5aFPoWe7/chO+lUsz875Aypcs6tmAjIGzB2zrHr6dYcB/7UPSWr2olMokd59AnQG0AaYBnwGtnWEqH9sWdp7R0zdSo1wJZt7fnoqlPZw0IsLgq6Hw3y4QcdS2/NFimCYzpVSWuFvH5GeMCXB5/7uI/OVmmSoX7Tp+gbs/30CF0kWZPTaQamVLeC6YxETYPB1+fx5MAvR4VhsTVkplm7sJbauItDPGbAQQkTbAWvfDUrkhJDyKuz9fT8miXnx1fwdqli/puWDiY2HWQAj9A+p3tY0JV8rHLfkrpfI9dxNaa2C9iCTdtl8f2CEiWwBjjGntZvkqhxw6c5GRn60DhK/GBlKnUinPBOLamHCtlhBwJ7S62w5TSik3uJvQ+uVIFCpXHT1/iRHT1hMbn8jcBzrSoGoZzwRyIhh+mgC3TIHaraHXK56JQylVKLmV0Iwx+0WkOfYuR4A/jDE73A9L5ZTwCzGMnLaOCzFxzBnbgetrlM37IOIvw8q3YNU7ULIiXDqb9zEopQo9d7uPGQ98A9R1/r4RkYdzIjDlvjNRlxn52XpORV5mxn3taV67fN4HcWSjvXtx5RTwHwLjNkDDm/M+DqVUoeduleMDQHtjTBSAiLwGrAE+djcw5Z6I6Dju/nwDh89GM+O+9rSuW9EzgRxYDpejYOQ8aNTTMzEopa4J7iY0AeJc3sc5w5QHRcbEcc/0DYSERzHt3rZ0aJDH3ascWA6JCdDwJug8AQL/oY0JK6VyXXYbJ/Y2xsQDM4F1IvKdM2oA+mC1R12KTWBM0CZ2HI3gk7va0LVx1Tyc+Xn47WnYMgt8b7QJzauo/VNKqVyW3TO0DUBrY8wUEVkG3Ig9M3sw6Zk0lfdi4hIY++UmNh06y3vDWtHTr3rezXzXT/Dz43DxFNwwAbo9mXfzVkopsp/QkqsVnQSmSczDYuMTeXj2n6wKOc1bQ1pwe4taeTfzg3/A1yOhhj+M+No+X6aUUnksuwmtqohMTGukMeadbJarsiE+IZEJX29h6e5wXh3QnMFtfHJ/psbAmf1QpaFtTHjgZ9Csv1YvKqU8Jru37XsBZYCyafypPJKYaJg0bxsLt5/g2b5+jAysl/szPX8EZg92GhMOs618BAzRZKaU8qjsnqEdN8a8lKORqCwzxvD0/O18v+Uok3pfz5jOudwWYmIibPocFr9gz9Bufh7K5mHVplJKpcPta2jKM4wxvPjjTuZsOML47g0Z171h7s4wPha+7AeH10CD7rYx4Yp5cDaolFKZlN2EdkcK0UQAAA+QSURBVFOORqGyxBjDlEV7CFoTypjO9Xm8V+PcnNnfjQn7tIVWd0HLEdqYsFIq38nWNTRjjDbG50EfLA3hk+X7GRlYl2dua4rkVnI5vg2m9YCjm+37Xi9Dq5GazJRS+ZK7LYWoPDZ15X7e+X0vg1r78HK/5rmTzOJibNuLq/4DpSrbB6aVUiqf04RWgMxcG8prC3fTN6AmUwYHUKRILiSzw+tgwXg4sw9ajrRdvJSqlPPzUUqpHKYJrYD4ZtMRnl2wg55+1Xl3aEu8ciOZge1BOv4y3PU/23SVUkoVEG51H5NdItJHRPaISIiIXNVGkohMFJGdIrJNRJaIyDV9O92CrUeZ/N02bmxUhQ9HtKKoVw7/bCFLIGSxfX3DBHh4rSYzpVSBk+cJTUS8gI+AWwA/YLiI+KWYbAvQ1hgTAMwDpuRtlPnHr8EnmPjNX7T3rcTUu9tS3Nsr5wqPPgvzH4ZZA2H1e3aYV1Eo7qEerZVSyg2eOENrD4QYYw4YY2KBuUA/1wmMMcuMMdHO23VAHrTllP8s2xP+/+3de3RV5ZnH8e9DwiUiQkFuAyggF0VAKTHBekFqrFRRbNVKxaAUsFZldNXRsUvb8VpH7djOWGu5RBAErEoFOkWoLrRKkVykCHKzqYIEdQKEiwghhDzzx94pxzRcEk6yk3N+n7Wy2Gdf3vMke4Vf3r3Pfl8mzlnBwK6tybnpHNKaxTHM1s6HZzLh/Rfhgrvg+pfj17aISASiuIfWBdgc87oIyDzC/uOA1w630cxuJpholFNOOSUe9TUIywq3ccvM9+jTsRXTx2ZwYvM4nqqP34aXxkCngXDDXOg8MH5ti4hEJIpAq+7TDF7tjmY3AOnA0MM15u6TgckA6enp1bbT2BRsLGH8jAK6t2vJzHGZtE6LwxiJ7rDtb9C+TzBX2dU50G+kxl8UkYQRxSXHIqBbzOuuwKdVdzKzLOA+4Ep3319PtUVuVdFOxk7Lp9NJLZg5PoO2LZsdf6M7NgX3ySZfBLu2BA9GD7hGYSYiCSWKHlo+0NvMegBbgFHA9bE7mNkgYBIw3N2L67/EaKz7bDfZOXm0admUWRMy6dCqxfE1WFEB+VPgjQeDELvkQWjVOT7Fiog0MPUeaO5ebma3A4sJpqF5zt3XmNlDQIG7LwCeJJie5uVwJIxP3P3K+q61PhUW7yE7J5e0pinMHj+Ezq3Tjq/B8jJ4/grYvBx6ZcGIX0Gbbkc/TkSkkYrkwWp3XwgsrLLuZzHLWfVeVIQ2bf+S0VOXA8bsCZl0a3tC7RuLHUz41HMhfSwMvE7jL4pIwovkwWo5ZMvOfVw/JZey8gpmjc+kZ/vjeAbs05XBfbLKwYSzHoCzRinMRCQpaOirCBXvLmX0lOXsLj3AnAlD6NuplpN9H9gHb/0nLHsaWp4MpbvjW6iISCOgQIvI9j37GT01l+Iv9jNzXCb9u7SuXUOblsGCibC9EAZlB1O8pH0tvsWKiDQCCrQI7Np7gOycPD4p2cvzP8hg8KnHEUCblsHBMsieB6cNi1+RIiKNjAKtnn1ReoAx0/IoLN7DlBvTGdKzXc0b+Vs4kHDvLDjvDsi8ReMvikjSU6DVo71l5YybXsCaLbt49obBDO3TvoYNlMCin8CqF6HH0CDQUprqAWkRERRo9ab0wEFunvEeBZtK+O9Rg7ikX8djP9gd1s6DhXfDvh1w4T1w4b/VXbEiIo2QAq0elJVXcOusFSwt3MYvrj2LK876l5o1sPEdePkm6Hw2ZL8KnQbUSZ0iIo2ZAq2OlR+s4M7f/ZUl64t55Kr+XDP4GGfCcYetG6DD6cFgwtdMgzOuhBSdMhGR6ujB6jpUUeHc/coqFq7+nPsvP4MbhhzjxNslH8OMkTBlGOwqCh6M7v9dhZmIyBHof8g64u7cN281r/51C3df2pfxF/Q8+kEVByF3Eix5GCwFLn0UWtXw8qSISJJSoNUBd+fBP6xlTt5mbh/Wi9uG9Tr6QeX7YfoIKMqD3pfCiF9C6y51X6yISIJQoMWZu/PE4g1MX7aRcef34K5v9TnaAeFgws2h+/mQcXMwV5nGXxQRqRHdQ4uzp5cU8uxbf2d05incf/kZ2JGCacsKmHQhFBUEr7P+AwZeqzATEakF9dDiaPLbf+ep1z/ku1/vwsMj+x8+zMr2wluPwbu/hhM7Qtme+i1URCQBKdDiZOa7G/n5wvVcPrAzT1w9kCZNDhNmG5cGgwmXfARfvxEueQjS2tRrrSIiiUiBFgcvFWzmp/PXkHVGR3513dmkphzhSu7mXPAKGLMAeg6tvyJFRBKcAu04zV+5hX+fu4oLep/MM6MH0bS6MPtwMWDQ51vwjX8NBhNu1rLeaxURSWQKtOOw6IPP+fFL75PRvS2Ts9Npnpry1R2+3AaL7oXVL0PPYUGgaTBhEZE6oUCrpTc3FDNxzgoGdm1Nzk3nkNYsJszc4YO58No9wezRF/0Ezv9xdMWKiCQBBVotLCvcxi0z36NPx1ZMH5vBic2r/Bg3vgNzx0GXwXDlr6Fjv2gKFRFJIgq0GirYWML4GQWc2u4EZo7LpHVaePnQHbauhw5nBIMJf28GnD4CmqQcuUEREYkLPVhdA6uKdjJ2Wj6dTmrBC+MzaduyWbCh5CN4/gqY8k3YtSV4MLrfSIWZiEg9Ug/tGK37bDfZOXm0admUWRMy6dCqRTCY8PJnYckjwQc9hj8GJ2kwYRGRKCjQjkFh8R6yc3JJa5rC7PFD6Nw6LRxM+HIoyoc+34YRTynMREQipEA7ik3bv2T01OWAMXtCJt3atAg2pDaHHkODZ8r6X63xF0VEIqZ7aEewZec+rp+SS1l5BbPGZ9Jz/3qYdAFszg92uPinGhlfRKSBUKAdRvHuUkZPWc7u0gO8MGYAfVc+BlOzYG8JlO+LujwREalClxyrsX3PfkZPzaX4i/28etlB+s4fDjs2QvoPIOtBaHFS1CWKiEgVCrQqdu09QHZOHp+U7GX62Az6fjYDrAnc9MdgAk4REWmQFGgxvig9wJhpeXTb+mcev7gPA05rB90nQsYPodkJUZcnIiJHENk9NDMbbmYbzKzQzO6tZntzM/tduD3XzLrXZT17y8q587k3GP/5w0xKfZIBm2cHG1JSFWYiIo1AJD00M0sBngEuAYqAfDNb4O5rY3YbB+xw915mNgp4HLiuLuopLStn+m+f4Mntz9AmdT9cdB+cd2ddvJWIiNSRqHpoGUChu3/k7mXAi8DIKvuMBJ4Pl18BLjaL/+fjDxys4Omc57i15HG87Wk0+dFSGHoPpDaL91uJiEgdiirQugCbY14Xheuq3cfdy4FdQLuqDZnZzWZWYGYFW7durXEhqU2Msm7n8edBv6TdxDehw+k1bkNERKIX1YdCqutpeS32wd0nA5MB0tPT/2n7UQsx474RZwJn1vRQERFpQKLqoRUB3WJedwU+Pdw+ZpYKtAZK6qU6ERFpdKIKtHygt5n1MLNmwChgQZV9FgA3hsvXAEvcvcY9MBERSQ6RXHJ093Izux1YDKQAz7n7GjN7CChw9wVADjDTzAoJemajoqhVREQah8gerHb3hcDCKut+FrNcClxb33WJiEjjpMGJRUQkISjQREQkISjQREQkISjQREQkIVgifRLezLYCm2p5+MnAtjiWI8dP56Rh0nlpeI73nJzq7u3jVUxUEirQjoeZFbh7etR1yCE6Jw2TzkvDo3MS0CVHERFJCAo0ERFJCAq0QyZHXYD8E52ThknnpeHROUH30EREJEGohyYiIglBgSYiIgkh6QPNzIab2QYzKzSze6OuR8DMnjOzYjP7IOpaJGBm3czsTTNbZ2ZrzOyOqGsSMLMWZpZnZu+H5+XBqGuKUlLfQzOzFOBD4BKCCUXzge+7+9pIC0tyZnYhsAeY4e79o65HwMw6A53dfYWZtQLeA67S70q0zMyAlu6+x8yaAkuBO9x9ecSlRSLZe2gZQKG7f+TuZcCLwMiIa0p67v42mp28QXH3z9x9Rbj8BbAO6BJtVeKBPeHLpuFX0vZSkj3QugCbY14XoV9SkSMys+7AICA32koEgitNZrYSKAZed/ekPS/JHmhWzbqk/etG5GjM7ERgLnCnu++Ouh4Bdz/o7mcDXYEMM0vay/TJHmhFQLeY112BTyOqRaRBC+/RzAVmufvvo65HvsrddwJvAcMjLiUyyR5o+UBvM+thZs2AUcCCiGsSaXDCDx/kAOvc/amo65GAmbU3szbhchqQBayPtqroJHWguXs5cDuwmOAm90vuvibaqsTM5gDvAn3NrMjMxkVdk3AekA1808xWhl+XRV2U0Bl408xWEfyB/rq7/2/ENUUmqT+2LyIiiSOpe2giIpI4FGgiIpIQFGgiIpIQFGgiIpIQFGgiIpIQFGiSsMysXcxHzD83sy0xr5vVsK1pZtb3GPfNMjM3s2/HrFtkZufX9Hs4TPtFlc8eicghqVEXIFJX3H07cDaAmT0A7HH3X9SyrbE1PGQzcD/wWm3er66YWWr4/KVIwlEPTZKSmd1jZh+EXxPDdb3COaVmmtlqM3spHH0BM1tqZpXheLmZrQjnoPrTYd5iBVBqZsOqee9/9LDMbIiZvREuP2Jm083sT2a20cyuMrP/Cmv8o5nF/gF6bzgPVq6Z9QyP72hmvzezgnDbkJh2J5nZ68C0+PwERRoeBZokHTPLAEYTTB90LnCrmQ0MN/cDnnH3AUAp8MMqx3YCngW+4+5nEQyXdjiPEvTSaqIHcBlwNTAbWBTOCVfBV8fo2+HuGcAkoHIoqv8BnnD3dOB7wNSY/QcBV7h7dg3rEWk0FGiSjC4A5rr73nBur3lA5f2tj2MmR3whZn2lc4E33X0TgLsfdt42d18CtDCzc2tQ28LwkuDqsI3Xw/Wrge4x+80J/50FfCNczgJ+G04lMg/4WmUPE5jv7qU1qEOk0dE9NElG1U0bVKnqWHBVX1s1647kUeC+KuvKOfTHZIsq2/aH/1YAZTHrK/jq72t1NRiQEU5We2ilGcCXx16ySOOkHpoko7eB75hZWji/10jgnXBbDzM7J1z+PsGU9rH+QjBA76kAZtb2SG/k7guBTsCZMas3AoPD5atr+T1cF1PjX8LlN4DbKneovOcnkiwUaJJ03D2P4JJdPrAceNbdV4eb1wATwtHLWwKTqxz7f8CPgPlm9j7BJb+j+TnBXHuVHgB+Y2bv8NVeWE2cYGZ5YS13hetuA84zs1VmthaYUMu2RRoljbYvEjKzXsAr4ey/ItLIqIcmIiIJQT00ERFJCOqhiYhIQlCgiYhIQlCgiYhIQlCgiYhIQlCgiYhIQvh/AHF/deEsdHAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "corpus_1= [id2word.doc2bow(word_tokenize(overall_reviews[2]))]\n",
    "temp_model = lda_model[corpus_1]\n",
    "y = [0,0,0,0]\n",
    "x = [0,1,2,3]\n",
    "yh = [0,0.3333333,0.66666666,1]\n",
    "y[1] = temp_model[0][0][1]\n",
    "y[2] = temp_model[0][1][1]\n",
    "y[3] = temp_model[0][2][1]\n",
    "fair_y_cum = np.cumsum(y)\n",
    "plt.plot(x,fair_y_cum,'-',x,yh,'--')\n",
    "plt.legend([\"Real Model Curve\",\"Ideal Model Curve\"])\n",
    "#fair_area = np.trapz(fair_y_cum, dx=4)\n",
    "plt.ylabel(\"Topic Distribution %\")\n",
    "plt.xlabel(\"Topic Number\")\n",
    "plt.title(\"CAP Curve of Topic Distribution VS. Three Topics for A Sample Review\")\n",
    "plt.xticks(np.arange(0, 4, step=1))\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.09159707895169805"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import statistics as sta\n",
    "np.mean(BB['GINI coe'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>star</th>\n",
       "      <th>sku</th>\n",
       "      <th>label</th>\n",
       "      <th>topic 0</th>\n",
       "      <th>topic 1</th>\n",
       "      <th>topic 2</th>\n",
       "      <th>topic 3</th>\n",
       "      <th>area</th>\n",
       "      <th>GINI coe</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>the bag would great deal available for store p...</td>\n",
       "      <td>1</td>\n",
       "      <td>8566349</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.031136</td>\n",
       "      <td>0.926000</td>\n",
       "      <td>0.042885</td>\n",
       "      <td>5.953133</td>\n",
       "      <td>0.007810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>guy cant say headoverheels for dirty dancing b...</td>\n",
       "      <td>3</td>\n",
       "      <td>6083463</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.032270</td>\n",
       "      <td>0.937522</td>\n",
       "      <td>0.030212</td>\n",
       "      <td>6.008256</td>\n",
       "      <td>-0.001377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>ordered this dvd cyber mondaybr best buy where...</td>\n",
       "      <td>5</td>\n",
       "      <td>8977805</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.024979</td>\n",
       "      <td>0.952785</td>\n",
       "      <td>0.022238</td>\n",
       "      <td>6.010981</td>\n",
       "      <td>-0.001831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>will continue order bestbuycom the movie not a...</td>\n",
       "      <td>4</td>\n",
       "      <td>3873909</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.081162</td>\n",
       "      <td>0.356197</td>\n",
       "      <td>0.562583</td>\n",
       "      <td>4.073965</td>\n",
       "      <td>0.321005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>this movie not one would have typically checke...</td>\n",
       "      <td>5</td>\n",
       "      <td>1067784</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.948321</td>\n",
       "      <td>0.022399</td>\n",
       "      <td>0.029266</td>\n",
       "      <td>9.676142</td>\n",
       "      <td>-0.612692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>71</td>\n",
       "      <td>this season babylon had the best season much h...</td>\n",
       "      <td>5</td>\n",
       "      <td>6074302</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.015393</td>\n",
       "      <td>0.968313</td>\n",
       "      <td>0.016303</td>\n",
       "      <td>5.996410</td>\n",
       "      <td>0.000597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>72</td>\n",
       "      <td>thank goodness tnt advertised this being relea...</td>\n",
       "      <td>5</td>\n",
       "      <td>8700015</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.011008</td>\n",
       "      <td>0.978343</td>\n",
       "      <td>0.010643</td>\n",
       "      <td>6.001423</td>\n",
       "      <td>-0.000238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>73</td>\n",
       "      <td>can just say quotits about timequot have been ...</td>\n",
       "      <td>5</td>\n",
       "      <td>8700015</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.021367</td>\n",
       "      <td>0.956504</td>\n",
       "      <td>0.022128</td>\n",
       "      <td>5.996947</td>\n",
       "      <td>0.000508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>74</td>\n",
       "      <td>bought this christmas gift for family with fou...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.018154</td>\n",
       "      <td>0.953994</td>\n",
       "      <td>0.027857</td>\n",
       "      <td>5.961216</td>\n",
       "      <td>0.006463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75</td>\n",
       "      <td>stargate the entire series just amazing able w...</td>\n",
       "      <td>5</td>\n",
       "      <td>8553399</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.024106</td>\n",
       "      <td>0.950166</td>\n",
       "      <td>0.025714</td>\n",
       "      <td>5.993484</td>\n",
       "      <td>0.001085</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>76 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              content  star      sku  label  \\\n",
       "0   the bag would great deal available for store p...     1  8566349      0   \n",
       "1   guy cant say headoverheels for dirty dancing b...     3  6083463      0   \n",
       "2   ordered this dvd cyber mondaybr best buy where...     5  8977805      0   \n",
       "3   will continue order bestbuycom the movie not a...     4  3873909      0   \n",
       "4   this movie not one would have typically checke...     5  1067784      0   \n",
       "..                                                ...   ...      ...    ...   \n",
       "71  this season babylon had the best season much h...     5  6074302      0   \n",
       "72  thank goodness tnt advertised this being relea...     5  8700015      0   \n",
       "73  can just say quotits about timequot have been ...     5  8700015      0   \n",
       "74  bought this christmas gift for family with fou...     5  3792604      0   \n",
       "75  stargate the entire series just amazing able w...     5  8553399      0   \n",
       "\n",
       "    topic 0   topic 1   topic 2   topic 3      area  GINI coe  \n",
       "0         0  0.031136  0.926000  0.042885  5.953133  0.007810  \n",
       "1         0  0.032270  0.937522  0.030212  6.008256 -0.001377  \n",
       "2         0  0.024979  0.952785  0.022238  6.010981 -0.001831  \n",
       "3         0  0.081162  0.356197  0.562583  4.073965  0.321005  \n",
       "4         0  0.948321  0.022399  0.029266  9.676142 -0.612692  \n",
       "..      ...       ...       ...       ...       ...       ...  \n",
       "71        0  0.015393  0.968313  0.016303  5.996410  0.000597  \n",
       "72        0  0.011008  0.978343  0.010643  6.001423 -0.000238  \n",
       "73        0  0.021367  0.956504  0.022128  5.996947  0.000508  \n",
       "74        0  0.018154  0.953994  0.027857  5.961216  0.006463  \n",
       "75        0  0.024106  0.950166  0.025714  5.993484  0.001085  \n",
       "\n",
       "[76 rows x 10 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "online = BB[BB.label==1] #83 rows\n",
    "store = BB[BB.label==0] # 76 rows\n",
    "store = store.reset_index()\n",
    "store = store.drop(columns=['index'])\n",
    "store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>star</th>\n",
       "      <th>sku</th>\n",
       "      <th>label</th>\n",
       "      <th>topic 0</th>\n",
       "      <th>topic 1</th>\n",
       "      <th>topic 2</th>\n",
       "      <th>topic 3</th>\n",
       "      <th>area</th>\n",
       "      <th>GINI coe</th>\n",
       "      <th>norm_GINI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>the bag would great deal available for store p...</td>\n",
       "      <td>1</td>\n",
       "      <td>8566349</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.031136</td>\n",
       "      <td>0.926000</td>\n",
       "      <td>0.042885</td>\n",
       "      <td>5.953133</td>\n",
       "      <td>0.007810</td>\n",
       "      <td>0.007810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>guy cant say headoverheels for dirty dancing b...</td>\n",
       "      <td>3</td>\n",
       "      <td>6083463</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.032270</td>\n",
       "      <td>0.937522</td>\n",
       "      <td>0.030212</td>\n",
       "      <td>6.008256</td>\n",
       "      <td>-0.001377</td>\n",
       "      <td>-0.001377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>ordered this dvd cyber mondaybr best buy where...</td>\n",
       "      <td>5</td>\n",
       "      <td>8977805</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.024979</td>\n",
       "      <td>0.952785</td>\n",
       "      <td>0.022238</td>\n",
       "      <td>6.010981</td>\n",
       "      <td>-0.001831</td>\n",
       "      <td>-0.001831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>will continue order bestbuycom the movie not a...</td>\n",
       "      <td>4</td>\n",
       "      <td>3873909</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.081162</td>\n",
       "      <td>0.356197</td>\n",
       "      <td>0.562583</td>\n",
       "      <td>4.073965</td>\n",
       "      <td>0.321005</td>\n",
       "      <td>0.321005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>this movie not one would have typically checke...</td>\n",
       "      <td>5</td>\n",
       "      <td>1067784</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.948321</td>\n",
       "      <td>0.022399</td>\n",
       "      <td>0.029266</td>\n",
       "      <td>9.676142</td>\n",
       "      <td>-0.612692</td>\n",
       "      <td>-0.612692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>71</td>\n",
       "      <td>this season babylon had the best season much h...</td>\n",
       "      <td>5</td>\n",
       "      <td>6074302</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.015393</td>\n",
       "      <td>0.968313</td>\n",
       "      <td>0.016303</td>\n",
       "      <td>5.996410</td>\n",
       "      <td>0.000597</td>\n",
       "      <td>0.000597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>72</td>\n",
       "      <td>thank goodness tnt advertised this being relea...</td>\n",
       "      <td>5</td>\n",
       "      <td>8700015</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.011008</td>\n",
       "      <td>0.978343</td>\n",
       "      <td>0.010643</td>\n",
       "      <td>6.001423</td>\n",
       "      <td>-0.000238</td>\n",
       "      <td>0.000135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>73</td>\n",
       "      <td>can just say quotits about timequot have been ...</td>\n",
       "      <td>5</td>\n",
       "      <td>8700015</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.021367</td>\n",
       "      <td>0.956504</td>\n",
       "      <td>0.022128</td>\n",
       "      <td>5.996947</td>\n",
       "      <td>0.000508</td>\n",
       "      <td>0.000135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>74</td>\n",
       "      <td>bought this christmas gift for family with fou...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.018154</td>\n",
       "      <td>0.953994</td>\n",
       "      <td>0.027857</td>\n",
       "      <td>5.961216</td>\n",
       "      <td>0.006463</td>\n",
       "      <td>0.006463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75</td>\n",
       "      <td>stargate the entire series just amazing able w...</td>\n",
       "      <td>5</td>\n",
       "      <td>8553399</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.024106</td>\n",
       "      <td>0.950166</td>\n",
       "      <td>0.025714</td>\n",
       "      <td>5.993484</td>\n",
       "      <td>0.001085</td>\n",
       "      <td>0.001085</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>76 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              content  star      sku  label  \\\n",
       "0   the bag would great deal available for store p...     1  8566349      0   \n",
       "1   guy cant say headoverheels for dirty dancing b...     3  6083463      0   \n",
       "2   ordered this dvd cyber mondaybr best buy where...     5  8977805      0   \n",
       "3   will continue order bestbuycom the movie not a...     4  3873909      0   \n",
       "4   this movie not one would have typically checke...     5  1067784      0   \n",
       "..                                                ...   ...      ...    ...   \n",
       "71  this season babylon had the best season much h...     5  6074302      0   \n",
       "72  thank goodness tnt advertised this being relea...     5  8700015      0   \n",
       "73  can just say quotits about timequot have been ...     5  8700015      0   \n",
       "74  bought this christmas gift for family with fou...     5  3792604      0   \n",
       "75  stargate the entire series just amazing able w...     5  8553399      0   \n",
       "\n",
       "    topic 0   topic 1   topic 2   topic 3      area  GINI coe  norm_GINI  \n",
       "0         0  0.031136  0.926000  0.042885  5.953133  0.007810   0.007810  \n",
       "1         0  0.032270  0.937522  0.030212  6.008256 -0.001377  -0.001377  \n",
       "2         0  0.024979  0.952785  0.022238  6.010981 -0.001831  -0.001831  \n",
       "3         0  0.081162  0.356197  0.562583  4.073965  0.321005   0.321005  \n",
       "4         0  0.948321  0.022399  0.029266  9.676142 -0.612692  -0.612692  \n",
       "..      ...       ...       ...       ...       ...       ...        ...  \n",
       "71        0  0.015393  0.968313  0.016303  5.996410  0.000597   0.000597  \n",
       "72        0  0.011008  0.978343  0.010643  6.001423 -0.000238   0.000135  \n",
       "73        0  0.021367  0.956504  0.022128  5.996947  0.000508   0.000135  \n",
       "74        0  0.018154  0.953994  0.027857  5.961216  0.006463   0.006463  \n",
       "75        0  0.024106  0.950166  0.025714  5.993484  0.001085   0.001085  \n",
       "\n",
       "[76 rows x 11 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "store_sku = [8946929,8534124,8829155,5998654,7865115,\n",
    "             8399146,9325606,9325544,8811045,9658199,\n",
    "             8700015]\n",
    "store['norm_GINI'] = store['GINI coe']\n",
    "for i in store_sku:\n",
    "    temp = []\n",
    "    pos = []\n",
    "    for j in range(76):\n",
    "        if store.loc[j].at['sku'] == i:\n",
    "            temp.append(store.at[j,'GINI coe'])\n",
    "            pos.append(j)\n",
    "    mean = sta.mean(temp)\n",
    "    m = 0\n",
    "    for k in pos:\n",
    "        store.at[k,\"norm_GINI\"] = mean\n",
    "        m += 1    \n",
    "store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sorting by first name \n",
    "store.sort_values(\"sku\", inplace=True) \n",
    "# dropping duplicate values \n",
    "store = store.drop_duplicates(subset='norm_GINI', keep='first')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>star</th>\n",
       "      <th>sku</th>\n",
       "      <th>label</th>\n",
       "      <th>topic 0</th>\n",
       "      <th>topic 1</th>\n",
       "      <th>topic 2</th>\n",
       "      <th>topic 3</th>\n",
       "      <th>area</th>\n",
       "      <th>GINI coe</th>\n",
       "      <th>norm_GINI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>this movie not one would have typically checke...</td>\n",
       "      <td>5</td>\n",
       "      <td>1067784</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.948321</td>\n",
       "      <td>0.022399</td>\n",
       "      <td>0.029266</td>\n",
       "      <td>9.676142</td>\n",
       "      <td>-0.612692</td>\n",
       "      <td>-0.612692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70</td>\n",
       "      <td>vampire diary the complete first season christ...</td>\n",
       "      <td>5</td>\n",
       "      <td>1085331</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.024928</td>\n",
       "      <td>0.026098</td>\n",
       "      <td>0.948978</td>\n",
       "      <td>2.303823</td>\n",
       "      <td>0.616029</td>\n",
       "      <td>0.616029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>63</td>\n",
       "      <td>the movie look stunning bluray and having the ...</td>\n",
       "      <td>5</td>\n",
       "      <td>1253546</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.017974</td>\n",
       "      <td>0.016054</td>\n",
       "      <td>0.965963</td>\n",
       "      <td>2.207986</td>\n",
       "      <td>0.632002</td>\n",
       "      <td>0.632002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>for everyone whose life hit rock bottom they t...</td>\n",
       "      <td>4</td>\n",
       "      <td>3142306</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.009488</td>\n",
       "      <td>0.009899</td>\n",
       "      <td>0.980612</td>\n",
       "      <td>2.115502</td>\n",
       "      <td>0.647416</td>\n",
       "      <td>0.647416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>bought online for when went store pickup the d...</td>\n",
       "      <td>5</td>\n",
       "      <td>3687406</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.074274</td>\n",
       "      <td>0.188624</td>\n",
       "      <td>0.737112</td>\n",
       "      <td>3.348715</td>\n",
       "      <td>0.441880</td>\n",
       "      <td>0.441880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>69</td>\n",
       "      <td>easy ordering online then just stopping the st...</td>\n",
       "      <td>5</td>\n",
       "      <td>9460601</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.096388</td>\n",
       "      <td>0.101654</td>\n",
       "      <td>0.801966</td>\n",
       "      <td>3.177739</td>\n",
       "      <td>0.470376</td>\n",
       "      <td>0.470376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56</td>\n",
       "      <td>were excitedly awaiting the arrival this movie...</td>\n",
       "      <td>5</td>\n",
       "      <td>9506117</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.035719</td>\n",
       "      <td>0.038412</td>\n",
       "      <td>0.925871</td>\n",
       "      <td>2.439400</td>\n",
       "      <td>0.593433</td>\n",
       "      <td>0.593433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>great movie best buy great seller loved that b...</td>\n",
       "      <td>5</td>\n",
       "      <td>9615274</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.088241</td>\n",
       "      <td>0.837365</td>\n",
       "      <td>0.074422</td>\n",
       "      <td>6.055446</td>\n",
       "      <td>-0.009242</td>\n",
       "      <td>-0.009242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>61</td>\n",
       "      <td>this been great dvd set love the show the pric...</td>\n",
       "      <td>5</td>\n",
       "      <td>9658199</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.057740</td>\n",
       "      <td>0.881377</td>\n",
       "      <td>0.060874</td>\n",
       "      <td>5.987414</td>\n",
       "      <td>0.002097</td>\n",
       "      <td>-0.228978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>preordered this movie january didnt have drive...</td>\n",
       "      <td>5</td>\n",
       "      <td>9761535</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.879408</td>\n",
       "      <td>0.044840</td>\n",
       "      <td>0.075797</td>\n",
       "      <td>9.214712</td>\n",
       "      <td>-0.535787</td>\n",
       "      <td>-0.535787</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>61 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              content  star      sku  label  \\\n",
       "4   this movie not one would have typically checke...     5  1067784      0   \n",
       "70  vampire diary the complete first season christ...     5  1085331      0   \n",
       "63  the movie look stunning bluray and having the ...     5  1253546      0   \n",
       "12  for everyone whose life hit rock bottom they t...     4  3142306      0   \n",
       "23  bought online for when went store pickup the d...     5  3687406      0   \n",
       "..                                                ...   ...      ...    ...   \n",
       "69  easy ordering online then just stopping the st...     5  9460601      0   \n",
       "56  were excitedly awaiting the arrival this movie...     5  9506117      0   \n",
       "22  great movie best buy great seller loved that b...     5  9615274      0   \n",
       "61  this been great dvd set love the show the pric...     5  9658199      0   \n",
       "29  preordered this movie january didnt have drive...     5  9761535      0   \n",
       "\n",
       "    topic 0   topic 1   topic 2   topic 3      area  GINI coe  norm_GINI  \n",
       "4         0  0.948321  0.022399  0.029266  9.676142 -0.612692  -0.612692  \n",
       "70        0  0.024928  0.026098  0.948978  2.303823  0.616029   0.616029  \n",
       "63        0  0.017974  0.016054  0.965963  2.207986  0.632002   0.632002  \n",
       "12        0  0.009488  0.009899  0.980612  2.115502  0.647416   0.647416  \n",
       "23        0  0.074274  0.188624  0.737112  3.348715  0.441880   0.441880  \n",
       "..      ...       ...       ...       ...       ...       ...        ...  \n",
       "69        0  0.096388  0.101654  0.801966  3.177739  0.470376   0.470376  \n",
       "56        0  0.035719  0.038412  0.925871  2.439400  0.593433   0.593433  \n",
       "22        0  0.088241  0.837365  0.074422  6.055446 -0.009242  -0.009242  \n",
       "61        0  0.057740  0.881377  0.060874  5.987414  0.002097  -0.228978  \n",
       "29        0  0.879408  0.044840  0.075797  9.214712 -0.535787  -0.535787  \n",
       "\n",
       "[61 rows x 11 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>star</th>\n",
       "      <th>sku</th>\n",
       "      <th>label</th>\n",
       "      <th>topic 0</th>\n",
       "      <th>topic 1</th>\n",
       "      <th>topic 2</th>\n",
       "      <th>topic 3</th>\n",
       "      <th>area</th>\n",
       "      <th>GINI coe</th>\n",
       "      <th>norm_GINI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>bagit hold candy not included trick treat you ...</td>\n",
       "      <td>5</td>\n",
       "      <td>8566349</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.035076</td>\n",
       "      <td>0.038941</td>\n",
       "      <td>0.925987</td>\n",
       "      <td>2.436382</td>\n",
       "      <td>0.593936</td>\n",
       "      <td>0.593936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>loved the movie and played delivered ontime</td>\n",
       "      <td>5</td>\n",
       "      <td>6083463</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.533636</td>\n",
       "      <td>0.219605</td>\n",
       "      <td>0.246712</td>\n",
       "      <td>7.147409</td>\n",
       "      <td>-0.191236</td>\n",
       "      <td>-0.191236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>this movie great for all age shipping extremel...</td>\n",
       "      <td>5</td>\n",
       "      <td>8977805</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.097499</td>\n",
       "      <td>0.174463</td>\n",
       "      <td>0.728103</td>\n",
       "      <td>3.477978</td>\n",
       "      <td>0.420336</td>\n",
       "      <td>0.420336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>this movie seriously rock from begining end to...</td>\n",
       "      <td>5</td>\n",
       "      <td>3873909</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.012842</td>\n",
       "      <td>0.970246</td>\n",
       "      <td>0.016921</td>\n",
       "      <td>5.983736</td>\n",
       "      <td>0.002710</td>\n",
       "      <td>0.002710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>excellent dvd very fast shipping from best buy...</td>\n",
       "      <td>5</td>\n",
       "      <td>1067784</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.089407</td>\n",
       "      <td>0.499146</td>\n",
       "      <td>0.411458</td>\n",
       "      <td>4.711864</td>\n",
       "      <td>0.214689</td>\n",
       "      <td>0.214689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>78</td>\n",
       "      <td>ordered the series incluning disc paid for dis...</td>\n",
       "      <td>1</td>\n",
       "      <td>8700015</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.841970</td>\n",
       "      <td>0.075293</td>\n",
       "      <td>0.082737</td>\n",
       "      <td>9.036935</td>\n",
       "      <td>-0.506157</td>\n",
       "      <td>-0.506157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>79</td>\n",
       "      <td>hassle then buying from ebay and having wait f...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.311262</td>\n",
       "      <td>0.556638</td>\n",
       "      <td>0.132122</td>\n",
       "      <td>6.716689</td>\n",
       "      <td>-0.119449</td>\n",
       "      <td>-0.036683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80</td>\n",
       "      <td>our purchase christmas gift for our granddaugh...</td>\n",
       "      <td>5</td>\n",
       "      <td>3792604</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.023886</td>\n",
       "      <td>0.949697</td>\n",
       "      <td>0.026427</td>\n",
       "      <td>5.989891</td>\n",
       "      <td>0.001684</td>\n",
       "      <td>-0.036683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>81</td>\n",
       "      <td>this just quick note for anyone who concerned ...</td>\n",
       "      <td>4</td>\n",
       "      <td>3792604</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.049815</td>\n",
       "      <td>0.888806</td>\n",
       "      <td>0.061348</td>\n",
       "      <td>5.953688</td>\n",
       "      <td>0.007718</td>\n",
       "      <td>-0.036683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>82</td>\n",
       "      <td>very excited receive this set however once beg...</td>\n",
       "      <td>1</td>\n",
       "      <td>8553399</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.018526</td>\n",
       "      <td>0.962070</td>\n",
       "      <td>0.019398</td>\n",
       "      <td>5.996472</td>\n",
       "      <td>0.000587</td>\n",
       "      <td>0.000587</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>83 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              content  star      sku  label  \\\n",
       "0   bagit hold candy not included trick treat you ...     5  8566349      1   \n",
       "1         loved the movie and played delivered ontime     5  6083463      1   \n",
       "2   this movie great for all age shipping extremel...     5  8977805      1   \n",
       "3   this movie seriously rock from begining end to...     5  3873909      1   \n",
       "4   excellent dvd very fast shipping from best buy...     5  1067784      1   \n",
       "..                                                ...   ...      ...    ...   \n",
       "78  ordered the series incluning disc paid for dis...     1  8700015      1   \n",
       "79  hassle then buying from ebay and having wait f...     5  3792604      1   \n",
       "80  our purchase christmas gift for our granddaugh...     5  3792604      1   \n",
       "81  this just quick note for anyone who concerned ...     4  3792604      1   \n",
       "82  very excited receive this set however once beg...     1  8553399      1   \n",
       "\n",
       "    topic 0   topic 1   topic 2   topic 3      area  GINI coe  norm_GINI  \n",
       "0         0  0.035076  0.038941  0.925987  2.436382  0.593936   0.593936  \n",
       "1         0  0.533636  0.219605  0.246712  7.147409 -0.191236  -0.191236  \n",
       "2         0  0.097499  0.174463  0.728103  3.477978  0.420336   0.420336  \n",
       "3         0  0.012842  0.970246  0.016921  5.983736  0.002710   0.002710  \n",
       "4         0  0.089407  0.499146  0.411458  4.711864  0.214689   0.214689  \n",
       "..      ...       ...       ...       ...       ...       ...        ...  \n",
       "78        0  0.841970  0.075293  0.082737  9.036935 -0.506157  -0.506157  \n",
       "79        0  0.311262  0.556638  0.132122  6.716689 -0.119449  -0.036683  \n",
       "80        0  0.023886  0.949697  0.026427  5.989891  0.001684  -0.036683  \n",
       "81        0  0.049815  0.888806  0.061348  5.953688  0.007718  -0.036683  \n",
       "82        0  0.018526  0.962070  0.019398  5.996472  0.000587   0.000587  \n",
       "\n",
       "[83 rows x 11 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "online = BB[BB.label==1] #83 rows\n",
    "online = BB[BB.label==1] #83 rows\n",
    "online = online.reset_index()\n",
    "online = online.drop(columns=['index'])\n",
    "online_sku = [8946929,8534124,9615274,9761535,3699043,\n",
    "              7561033,8399146,8539575,9325544,8953395,\n",
    "              8811045,9460576,3792604]\n",
    "online['norm_GINI'] = online['GINI coe']\n",
    "for i in online_sku:\n",
    "    temp = []\n",
    "    pos = []\n",
    "    for j in range(len(online['norm_GINI'])):\n",
    "        if online.loc[j].at['sku'] == i:\n",
    "            temp.append(online.at[j,'GINI coe'])\n",
    "            pos.append(j)\n",
    "    mean = sta.mean(temp)\n",
    "    m = 0\n",
    "    for k in pos:\n",
    "        online.at[k,\"norm_GINI\"] = mean\n",
    "        m += 1    \n",
    "online"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>star</th>\n",
       "      <th>sku</th>\n",
       "      <th>label</th>\n",
       "      <th>topic 0</th>\n",
       "      <th>topic 1</th>\n",
       "      <th>topic 2</th>\n",
       "      <th>topic 3</th>\n",
       "      <th>area</th>\n",
       "      <th>GINI coe</th>\n",
       "      <th>norm_GINI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>excellent dvd very fast shipping from best buy...</td>\n",
       "      <td>5</td>\n",
       "      <td>1067784</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.089407</td>\n",
       "      <td>0.499146</td>\n",
       "      <td>0.411458</td>\n",
       "      <td>4.711864</td>\n",
       "      <td>0.214689</td>\n",
       "      <td>0.214689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>76</td>\n",
       "      <td>didnt think love this show but really all stro...</td>\n",
       "      <td>5</td>\n",
       "      <td>1085331</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.030585</td>\n",
       "      <td>0.942703</td>\n",
       "      <td>0.026712</td>\n",
       "      <td>6.015489</td>\n",
       "      <td>-0.002582</td>\n",
       "      <td>-0.002582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>67</td>\n",
       "      <td>this the best box set have received for any mo...</td>\n",
       "      <td>5</td>\n",
       "      <td>1253546</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.033900</td>\n",
       "      <td>0.915050</td>\n",
       "      <td>0.051047</td>\n",
       "      <td>5.931389</td>\n",
       "      <td>0.011434</td>\n",
       "      <td>0.011434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>expected entertaining but relieved laugh from ...</td>\n",
       "      <td>5</td>\n",
       "      <td>3142306</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.940072</td>\n",
       "      <td>0.028776</td>\n",
       "      <td>0.031150</td>\n",
       "      <td>9.635674</td>\n",
       "      <td>-0.605947</td>\n",
       "      <td>-0.605947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>this movie good story and can funny point but ...</td>\n",
       "      <td>3</td>\n",
       "      <td>3687406</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.304724</td>\n",
       "      <td>0.272687</td>\n",
       "      <td>0.422578</td>\n",
       "      <td>5.528520</td>\n",
       "      <td>0.078579</td>\n",
       "      <td>0.078579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75</td>\n",
       "      <td>course review going excellent have purchased s...</td>\n",
       "      <td>5</td>\n",
       "      <td>9460601</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.904446</td>\n",
       "      <td>0.058839</td>\n",
       "      <td>0.036710</td>\n",
       "      <td>9.470915</td>\n",
       "      <td>-0.578487</td>\n",
       "      <td>-0.578487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58</td>\n",
       "      <td>not clear that this movie spanish ordered onli...</td>\n",
       "      <td>1</td>\n",
       "      <td>9506117</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.041784</td>\n",
       "      <td>0.048444</td>\n",
       "      <td>0.909771</td>\n",
       "      <td>2.528048</td>\n",
       "      <td>0.578658</td>\n",
       "      <td>0.578658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>the movie very entertaining the whole experien...</td>\n",
       "      <td>5</td>\n",
       "      <td>9615274</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.398288</td>\n",
       "      <td>0.268157</td>\n",
       "      <td>0.333320</td>\n",
       "      <td>6.258461</td>\n",
       "      <td>-0.043078</td>\n",
       "      <td>-0.144637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>66</td>\n",
       "      <td>received dvd set very timely manner and very p...</td>\n",
       "      <td>5</td>\n",
       "      <td>9658199</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.039981</td>\n",
       "      <td>0.916267</td>\n",
       "      <td>0.043756</td>\n",
       "      <td>5.984921</td>\n",
       "      <td>0.002512</td>\n",
       "      <td>0.002512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>really love the twilight series and after orde...</td>\n",
       "      <td>5</td>\n",
       "      <td>9761535</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.088766</td>\n",
       "      <td>0.773914</td>\n",
       "      <td>0.137243</td>\n",
       "      <td>5.805631</td>\n",
       "      <td>0.032394</td>\n",
       "      <td>0.251176</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>61 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              content  star      sku  label  \\\n",
       "4   excellent dvd very fast shipping from best buy...     5  1067784      1   \n",
       "76  didnt think love this show but really all stro...     5  1085331      1   \n",
       "67  this the best box set have received for any mo...     5  1253546      1   \n",
       "12  expected entertaining but relieved laugh from ...     5  3142306      1   \n",
       "23  this movie good story and can funny point but ...     3  3687406      1   \n",
       "..                                                ...   ...      ...    ...   \n",
       "75  course review going excellent have purchased s...     5  9460601      1   \n",
       "58  not clear that this movie spanish ordered onli...     1  9506117      1   \n",
       "20  the movie very entertaining the whole experien...     5  9615274      1   \n",
       "66  received dvd set very timely manner and very p...     5  9658199      1   \n",
       "29  really love the twilight series and after orde...     5  9761535      1   \n",
       "\n",
       "    topic 0   topic 1   topic 2   topic 3      area  GINI coe  norm_GINI  \n",
       "4         0  0.089407  0.499146  0.411458  4.711864  0.214689   0.214689  \n",
       "76        0  0.030585  0.942703  0.026712  6.015489 -0.002582  -0.002582  \n",
       "67        0  0.033900  0.915050  0.051047  5.931389  0.011434   0.011434  \n",
       "12        0  0.940072  0.028776  0.031150  9.635674 -0.605947  -0.605947  \n",
       "23        0  0.304724  0.272687  0.422578  5.528520  0.078579   0.078579  \n",
       "..      ...       ...       ...       ...       ...       ...        ...  \n",
       "75        0  0.904446  0.058839  0.036710  9.470915 -0.578487  -0.578487  \n",
       "58        0  0.041784  0.048444  0.909771  2.528048  0.578658   0.578658  \n",
       "20        0  0.398288  0.268157  0.333320  6.258461 -0.043078  -0.144637  \n",
       "66        0  0.039981  0.916267  0.043756  5.984921  0.002512   0.002512  \n",
       "29        0  0.088766  0.773914  0.137243  5.805631  0.032394   0.251176  \n",
       "\n",
       "[61 rows x 11 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sorting by first name \n",
    "online.sort_values(\"sku\", inplace=True) \n",
    "# dropping duplicate values \n",
    "online = online.drop_duplicates(subset='norm_GINI', keep='first')\n",
    "online"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.stats as stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(store.dropna()['norm_GINI'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "#store.to_csv('/Users/jiaruizhang/Desktop/reviews/coding/store_topic_div.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60 60\n"
     ]
    }
   ],
   "source": [
    "store = store[store.sku != 8953527]\n",
    "online = online[online.sku != 8953527]\n",
    "print(len(online),len(store))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ttest_relResult(statistic=-0.6450838779690212, pvalue=0.5213734919438007)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats.ttest_rel(store['norm_GINI'], online['norm_GINI'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.07485406635284904"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sta.mean(store['norm_GINI'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.11605093420282352"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sta.mean(online['norm_GINI'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}