01_data_preparation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4a2a279f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import necessary packages\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from fastai.text.all import *\n",
    "import fastai\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e326cbf5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.7.7\n",
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "print(fastai.__version__)\n",
    "print(torch.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fdb6cb6a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3.3.1'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import spacy\n",
    "spacy.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "38c745db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "401c9723",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserName</th>\n",
       "      <th>ScreenName</th>\n",
       "      <th>Location</th>\n",
       "      <th>TweetAt</th>\n",
       "      <th>OriginalTweet</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3799</td>\n",
       "      <td>48751</td>\n",
       "      <td>London</td>\n",
       "      <td>16-03-2020</td>\n",
       "      <td>@MeNyrbie @Phil_Gahan @Chrisitv https://t.co/iFz9FAn2Pa and https://t.co/xX6ghGFzCC and https://t.co/I2NlzdxNo8</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3800</td>\n",
       "      <td>48752</td>\n",
       "      <td>UK</td>\n",
       "      <td>16-03-2020</td>\n",
       "      <td>advice Talk to your neighbours family to exchange phone numbers create contact list with phone numbers of neighbours schools employer chemist GP set up online shopping accounts if poss adequate supplies of regular meds but not over order</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3801</td>\n",
       "      <td>48753</td>\n",
       "      <td>Vagabonds</td>\n",
       "      <td>16-03-2020</td>\n",
       "      <td>Coronavirus Australia: Woolworths to give elderly, disabled dedicated shopping hours amid COVID-19 outbreak https://t.co/bInCA9Vp8P</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3802</td>\n",
       "      <td>48754</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16-03-2020</td>\n",
       "      <td>My food stock is not the only one which is empty...\\r\\r\\n\\r\\r\\nPLEASE, don't panic, THERE WILL BE ENOUGH FOOD FOR EVERYONE if you do not take more than you need. \\r\\r\\nStay calm, stay safe.\\r\\r\\n\\r\\r\\n#COVID19france #COVID_19 #COVID19 #coronavirus #confinement #Confinementotal #ConfinementGeneral https://t.co/zrlG0Z520j</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3803</td>\n",
       "      <td>48755</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16-03-2020</td>\n",
       "      <td>Me, ready to go at supermarket during the #COVID19 outbreak.\\r\\r\\n\\r\\r\\nNot because I'm paranoid, but because my food stock is litteraly empty. The #coronavirus is a serious thing, but please, don't panic. It causes shortage...\\r\\r\\n\\r\\r\\n#CoronavirusFrance #restezchezvous #StayAtHome #confinement https://t.co/usmuaLq72n</td>\n",
       "      <td>Extremely Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41152</th>\n",
       "      <td>44951</td>\n",
       "      <td>89903</td>\n",
       "      <td>Wellington City, New Zealand</td>\n",
       "      <td>14-04-2020</td>\n",
       "      <td>Airline pilots offering to stock supermarket shelves in #NZ lockdown #COVID-19 https://t.co/cz89uA0HNp</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41153</th>\n",
       "      <td>44952</td>\n",
       "      <td>89904</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14-04-2020</td>\n",
       "      <td>Response to complaint not provided citing COVID-19 related delays. Yet prompt in rejecting policy before consumer TAT is over. Way to go ?</td>\n",
       "      <td>Extremely Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41154</th>\n",
       "      <td>44953</td>\n",
       "      <td>89905</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14-04-2020</td>\n",
       "      <td>You know itÂs getting tough when @KameronWilds  is rationing toilet paper #coronavirus #toiletpaper @kroger martinsville, help us out!!</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41155</th>\n",
       "      <td>44954</td>\n",
       "      <td>89906</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14-04-2020</td>\n",
       "      <td>Is it wrong that the smell of hand sanitizer is starting to turn me on?\\r\\r\\n\\r\\r\\n#coronavirus #COVID19 #coronavirus</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41156</th>\n",
       "      <td>44955</td>\n",
       "      <td>89907</td>\n",
       "      <td>i love you so much || he/him</td>\n",
       "      <td>14-04-2020</td>\n",
       "      <td>@TartiiCat Well new/used Rift S are going for $700.00 on Amazon rn although the normal market price is usually $400.00 . Prices are really crazy right now for vr headsets since HL Alex was announced and it's only been worse with COVID-19. Up to you whethe</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>41157 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       UserName  ScreenName                      Location     TweetAt  \\\n",
       "0          3799       48751                        London  16-03-2020   \n",
       "1          3800       48752                            UK  16-03-2020   \n",
       "2          3801       48753                     Vagabonds  16-03-2020   \n",
       "3          3802       48754                           NaN  16-03-2020   \n",
       "4          3803       48755                           NaN  16-03-2020   \n",
       "...         ...         ...                           ...         ...   \n",
       "41152     44951       89903  Wellington City, New Zealand  14-04-2020   \n",
       "41153     44952       89904                           NaN  14-04-2020   \n",
       "41154     44953       89905                           NaN  14-04-2020   \n",
       "41155     44954       89906                           NaN  14-04-2020   \n",
       "41156     44955       89907  i love you so much || he/him  14-04-2020   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                            OriginalTweet  \\\n",
       "0                                                                                                                                                                                                                         @MeNyrbie @Phil_Gahan @Chrisitv https://t.co/iFz9FAn2Pa and https://t.co/xX6ghGFzCC and https://t.co/I2NlzdxNo8   \n",
       "1                                                                                           advice Talk to your neighbours family to exchange phone numbers create contact list with phone numbers of neighbours schools employer chemist GP set up online shopping accounts if poss adequate supplies of regular meds but not over order   \n",
       "2                                                                                                                                                                                                     Coronavirus Australia: Woolworths to give elderly, disabled dedicated shopping hours amid COVID-19 outbreak https://t.co/bInCA9Vp8P   \n",
       "3       My food stock is not the only one which is empty...\\r\\r\\n\\r\\r\\nPLEASE, don't panic, THERE WILL BE ENOUGH FOOD FOR EVERYONE if you do not take more than you need. \\r\\r\\nStay calm, stay safe.\\r\\r\\n\\r\\r\\n#COVID19france #COVID_19 #COVID19 #coronavirus #confinement #Confinementotal #ConfinementGeneral https://t.co/zrlG0Z520j   \n",
       "4      Me, ready to go at supermarket during the #COVID19 outbreak.\\r\\r\\n\\r\\r\\nNot because I'm paranoid, but because my food stock is litteraly empty. The #coronavirus is a serious thing, but please, don't panic. It causes shortage...\\r\\r\\n\\r\\r\\n#CoronavirusFrance #restezchezvous #StayAtHome #confinement https://t.co/usmuaLq72n   \n",
       "...                                                                                                                                                                                                                                                                                                                                   ...   \n",
       "41152                                                                                                                                                                                                                              Airline pilots offering to stock supermarket shelves in #NZ lockdown #COVID-19 https://t.co/cz89uA0HNp   \n",
       "41153                                                                                                                                                                                          Response to complaint not provided citing COVID-19 related delays. Yet prompt in rejecting policy before consumer TAT is over. Way to go ?   \n",
       "41154                                                                                                                                                                                            You know itÂs getting tough when @KameronWilds  is rationing toilet paper #coronavirus #toiletpaper @kroger martinsville, help us out!!   \n",
       "41155                                                                                                                                                                                                               Is it wrong that the smell of hand sanitizer is starting to turn me on?\\r\\r\\n\\r\\r\\n#coronavirus #COVID19 #coronavirus   \n",
       "41156                                                                     @TartiiCat Well new/used Rift S are going for $700.00 on Amazon rn although the normal market price is usually $400.00 . Prices are really crazy right now for vr headsets since HL Alex was announced and it's only been worse with COVID-19. Up to you whethe   \n",
       "\n",
       "                Sentiment  \n",
       "0                 Neutral  \n",
       "1                Positive  \n",
       "2                Positive  \n",
       "3                Positive  \n",
       "4      Extremely Negative  \n",
       "...                   ...  \n",
       "41152             Neutral  \n",
       "41153  Extremely Negative  \n",
       "41154            Positive  \n",
       "41155             Neutral  \n",
       "41156            Negative  \n",
       "\n",
       "[41157 rows x 6 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read traing data \n",
    "train_data = pd.read_csv(r\"E:\\\\Work\\\\Data_Science\\\\Projects\\\\Text_Classification_FastAI_V2\\\\archive\\\\Corona_NLP_train.csv\", encoding='latin-1')\n",
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f9f32a18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserName</th>\n",
       "      <th>ScreenName</th>\n",
       "      <th>Location</th>\n",
       "      <th>TweetAt</th>\n",
       "      <th>OriginalTweet</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>44953</td>\n",
       "      <td>NYC</td>\n",
       "      <td>02-03-2020</td>\n",
       "      <td>TRENDING: New Yorkers encounter empty supermarket shelves (pictured, Wegmans in Brooklyn), sold-out online grocers (FoodKick, MaxDelivery) as #coronavirus-fearing shoppers stock up https://t.co/Gr76pcrLWh https://t.co/ivMKMsqdT1</td>\n",
       "      <td>Extremely Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>44954</td>\n",
       "      <td>Seattle, WA</td>\n",
       "      <td>02-03-2020</td>\n",
       "      <td>When I couldn't find hand sanitizer at Fred Meyer, I turned to #Amazon. But $114.97 for a 2 pack of Purell??!!Check out how  #coronavirus concerns are driving up prices. https://t.co/ygbipBflMY</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>44955</td>\n",
       "      <td>NaN</td>\n",
       "      <td>02-03-2020</td>\n",
       "      <td>Find out how you can protect yourself and loved ones from #coronavirus. ?</td>\n",
       "      <td>Extremely Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>44956</td>\n",
       "      <td>Chicagoland</td>\n",
       "      <td>02-03-2020</td>\n",
       "      <td>#Panic buying hits #NewYork City as anxious shoppers stock up on food&amp;amp;medical supplies after #healthcare worker in her 30s becomes #BigApple 1st confirmed #coronavirus patient OR a #Bloomberg staged event?\\r\\r\\n\\r\\r\\nhttps://t.co/IASiReGPC4\\r\\r\\n\\r\\r\\n#QAnon #QAnon2018 #QAnon2020 \\r\\r\\n#Election2020 #CDC https://t.co/29isZOewxu</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>44957</td>\n",
       "      <td>Melbourne, Victoria</td>\n",
       "      <td>03-03-2020</td>\n",
       "      <td>#toiletpaper #dunnypaper #coronavirus #coronavirusaustralia #CoronaVirusUpdate #Covid_19 #9News  #Corvid19 #7NewsMelb #dunnypapergate #Costco    One week everyone buying baby milk powder the next everyone buying up toilet paper. https://t.co/ScZryVvsIh</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3793</th>\n",
       "      <td>3794</td>\n",
       "      <td>48746</td>\n",
       "      <td>Israel ??</td>\n",
       "      <td>16-03-2020</td>\n",
       "      <td>Meanwhile In A Supermarket in Israel -- People dance and sing together to stay positive. ? #CoronaVirus https://t.co/VLLqGI3R16</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3794</th>\n",
       "      <td>3795</td>\n",
       "      <td>48747</td>\n",
       "      <td>Farmington, NM</td>\n",
       "      <td>16-03-2020</td>\n",
       "      <td>Did you panic buy a lot of non-perishable items? ECHO needs food donations. It is also looking for volunteers to pack boxes. Volunteers will be screened to be sure they don't have COVID-19. https://t.co/TST4Vjhbu4</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3795</th>\n",
       "      <td>3796</td>\n",
       "      <td>48748</td>\n",
       "      <td>Haverford, PA</td>\n",
       "      <td>16-03-2020</td>\n",
       "      <td>Asst Prof of Economics @cconces was on @NBCPhiladelphia talking about her recent research on coronavirus' impact on the economy. Watch it here (starting at :33): https://t.co/8tfYNoro5l</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3796</th>\n",
       "      <td>3797</td>\n",
       "      <td>48749</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16-03-2020</td>\n",
       "      <td>Gov need to do somethings instead of biar je rakyat assume 'lockdown' ke or even worst. Harini semua supermarket crowded like hell. Lagi mudah virus tu tersebar ?? #COVID2019</td>\n",
       "      <td>Extremely Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3797</th>\n",
       "      <td>3798</td>\n",
       "      <td>48750</td>\n",
       "      <td>Arlington, Virginia</td>\n",
       "      <td>16-03-2020</td>\n",
       "      <td>I and @ForestandPaper members are committed to the safety of our employees and our end-users. We are monitoring COVID-19. Rest assured that tissue manufacturers are continuing to produce and ship products.  https://t.co/qF6hclCAEq https://t.co/xyvbNsFeXA</td>\n",
       "      <td>Extremely Positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3798 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      UserName  ScreenName             Location     TweetAt  \\\n",
       "0            1       44953                  NYC  02-03-2020   \n",
       "1            2       44954          Seattle, WA  02-03-2020   \n",
       "2            3       44955                  NaN  02-03-2020   \n",
       "3            4       44956          Chicagoland  02-03-2020   \n",
       "4            5       44957  Melbourne, Victoria  03-03-2020   \n",
       "...        ...         ...                  ...         ...   \n",
       "3793      3794       48746            Israel ??  16-03-2020   \n",
       "3794      3795       48747       Farmington, NM  16-03-2020   \n",
       "3795      3796       48748        Haverford, PA  16-03-2020   \n",
       "3796      3797       48749                  NaN  16-03-2020   \n",
       "3797      3798       48750  Arlington, Virginia  16-03-2020   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                      OriginalTweet  \\\n",
       "0                                                                                                              TRENDING: New Yorkers encounter empty supermarket shelves (pictured, Wegmans in Brooklyn), sold-out online grocers (FoodKick, MaxDelivery) as #coronavirus-fearing shoppers stock up https://t.co/Gr76pcrLWh https://t.co/ivMKMsqdT1   \n",
       "1                                                                                                                                                 When I couldn't find hand sanitizer at Fred Meyer, I turned to #Amazon. But $114.97 for a 2 pack of Purell??!!Check out how  #coronavirus concerns are driving up prices. https://t.co/ygbipBflMY   \n",
       "2                                                                                                                                                                                                                                                                         Find out how you can protect yourself and loved ones from #coronavirus. ?   \n",
       "3     #Panic buying hits #NewYork City as anxious shoppers stock up on food&amp;medical supplies after #healthcare worker in her 30s becomes #BigApple 1st confirmed #coronavirus patient OR a #Bloomberg staged event?\\r\\r\\n\\r\\r\\nhttps://t.co/IASiReGPC4\\r\\r\\n\\r\\r\\n#QAnon #QAnon2018 #QAnon2020 \\r\\r\\n#Election2020 #CDC https://t.co/29isZOewxu   \n",
       "4                                                                                      #toiletpaper #dunnypaper #coronavirus #coronavirusaustralia #CoronaVirusUpdate #Covid_19 #9News  #Corvid19 #7NewsMelb #dunnypapergate #Costco    One week everyone buying baby milk powder the next everyone buying up toilet paper. https://t.co/ScZryVvsIh   \n",
       "...                                                                                                                                                                                                                                                                                                                                             ...   \n",
       "3793                                                                                                                                                                                                                Meanwhile In A Supermarket in Israel -- People dance and sing together to stay positive. ? #CoronaVirus https://t.co/VLLqGI3R16   \n",
       "3794                                                                                                                          Did you panic buy a lot of non-perishable items? ECHO needs food donations. It is also looking for volunteers to pack boxes. Volunteers will be screened to be sure they don't have COVID-19. https://t.co/TST4Vjhbu4   \n",
       "3795                                                                                                                                                      Asst Prof of Economics @cconces was on @NBCPhiladelphia talking about her recent research on coronavirus' impact on the economy. Watch it here (starting at :33): https://t.co/8tfYNoro5l   \n",
       "3796                                                                                                                                                                 Gov need to do somethings instead of biar je rakyat assume 'lockdown' ke or even worst. Harini semua supermarket crowded like hell. Lagi mudah virus tu tersebar ?? #COVID2019   \n",
       "3797                                                                                 I and @ForestandPaper members are committed to the safety of our employees and our end-users. We are monitoring COVID-19. Rest assured that tissue manufacturers are continuing to produce and ship products.  https://t.co/qF6hclCAEq https://t.co/xyvbNsFeXA   \n",
       "\n",
       "               Sentiment  \n",
       "0     Extremely Negative  \n",
       "1               Positive  \n",
       "2     Extremely Positive  \n",
       "3               Negative  \n",
       "4                Neutral  \n",
       "...                  ...  \n",
       "3793            Positive  \n",
       "3794            Negative  \n",
       "3795             Neutral  \n",
       "3796  Extremely Negative  \n",
       "3797  Extremely Positive  \n",
       "\n",
       "[3798 rows x 6 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read test data\n",
    "test_data = pd.read_csv(r\"E:\\\\Work\\\\Data_Science\\\\Projects\\\\Text_Classification_FastAI_V2\\\\archive\\\\Corona_NLP_test.csv\", encoding='latin-1')\n",
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "53f1b881",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 41157 entries, 0 to 41156\n",
      "Data columns (total 6 columns):\n",
      " #   Column         Non-Null Count  Dtype \n",
      "---  ------         --------------  ----- \n",
      " 0   UserName       41157 non-null  int64 \n",
      " 1   ScreenName     41157 non-null  int64 \n",
      " 2   Location       32567 non-null  object\n",
      " 3   TweetAt        41157 non-null  object\n",
      " 4   OriginalTweet  41157 non-null  object\n",
      " 5   Sentiment      41157 non-null  object\n",
      "dtypes: int64(2), object(4)\n",
      "memory usage: 1.9+ MB\n"
     ]
    }
   ],
   "source": [
    "train_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4fc88996",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3798 entries, 0 to 3797\n",
      "Data columns (total 6 columns):\n",
      " #   Column         Non-Null Count  Dtype \n",
      "---  ------         --------------  ----- \n",
      " 0   UserName       3798 non-null   int64 \n",
      " 1   ScreenName     3798 non-null   int64 \n",
      " 2   Location       2964 non-null   object\n",
      " 3   TweetAt        3798 non-null   object\n",
      " 4   OriginalTweet  3798 non-null   object\n",
      " 5   Sentiment      3798 non-null   object\n",
      "dtypes: int64(2), object(4)\n",
      "memory usage: 178.2+ KB\n"
     ]
    }
   ],
   "source": [
    "test_data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b3aaa40",
   "metadata": {},
   "source": [
    "### Prepare data for training Language model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "83fb56a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to IPython and Windows limitation, python multiprocessing isn't available now.\n",
      "So `n_workers` has to be changed to 0 to avoid getting stuck\n",
      "CPU times: total: 30.2 s\n",
      "Wall time: 30.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from fastai.text.all import *\n",
    "\n",
    "fastai_language_model_data = (\n",
    "    TextDataLoaders.from_df(\n",
    "                            train_data,\n",
    "                            text_col='OriginalTweet',\n",
    "                            is_lm=True\n",
    "                            )\n",
    "                            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "152ddbbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(fastai_language_model_data, '..\\lm_data.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6ff9fcf",
   "metadata": {},
   "source": [
    "### Prepare training data for multi-class classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "abd9b510",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to IPython and Windows limitation, python multiprocessing isn't available now.\n",
      "So `n_workers` has to be changed to 0 to avoid getting stuck\n",
      "Due to IPython and Windows limitation, python multiprocessing isn't available now.\n",
      "So `number_workers` is changed to 0 to avoid getting stuck\n",
      "CPU times: total: 34.8 s\n",
      "Wall time: 34.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "fastai_text_classification_data= (\n",
    "    TextDataLoaders.from_df(\n",
    "        df = train_data,\n",
    "        valid_pct=0.3,\n",
    "        seed = 12345,\n",
    "        text_col = 'OriginalTweet',\n",
    "        label_col = 'Sentiment'\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "26d583ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>xxbos xxrep 5 ? ? ? xxrep 7 ? ? ? xxrep 7 ? xxrep 4 ? xxrep 4 ? xxrep 11 ? ? ? xxrep 6 ? xxrep 4 ? , xxrep 3 ? xxrep 3 ? ? ? xxrep 3 ? xxrep 4 ? xxrep 3 ? ? ? ? ? xxrep 4 ? ? ? xxrep 3 ? , xxrep 4 ? ? ? ? ? xxrep 6 ? xxrep 3 ? xxrep 3 ? xxrep 3 ? ? ? xxrep 3 ? \\r\\r\\n▁ xxrep 5 ? xxrep 6 ? ? ? xxrep 3 ? xxrep 4 ? xxrep 4 ? ? ? xxrep 4 ? xxrep 6 ? xxrep 4 ? xxrep 8 ? ? ? xxrep 6 ? ? ? xxrep 5 ? ? ? xxrep 3 ? xxrep 4 ? ? ? xxrep 7 ? xxrep 5 ? - xxrep 8 ? xxrep 5</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>xxbos ? xxrep 4 ? xxrep 5 ? xxrep 4 ? xxrep 3 ? xxrep 6 ? xxrep 3 ? xxrep 3 ? \\r\\r\\n\\r\\r\\n ? xxrep 5 ? xxrep 5 ? xxrep 6 ? ? ? ? xxrep 7 ? ? ? xxrep 4 ? ? ? ? xxrep 7 ? xxrep 4 ? xxrep 4 ? xxrep 3 ? xxrep 6 ? xxrep 5 ? xxrep 8 ? xxrep 4 ? xxrep 6 ? xxrep 5 ? ? xxrep 8 ? xxrep 7 ? xxrep 3 ? ? xxrep 4 ? ? xxrep 3 ? xxrep 3 ? ? ? xxrep 3 ? ? ? ? ? xxrep 4 ? ? xxrep 3 ? ? xxrep 4 ? xxrep 5 ? xxrep 4 ? xxrep 3 ? xxrep 6 ? xxrep 3 ? xxrep 3 ? . https : / / t.co / xxunk</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>xxbos xxrep 4 ? xxrep 9 ? ? ? 200 xxup ml xxrep 4 ? ? ? xxrep 5 ? xxrep 4 ? 100 ? ? . ? ? xxrep 4 ? xxrep 4 ? xxrep 5 ? xxrep 4 ? xxrep 4 ? ? ? xxrep 6 ? ? ? xxrep 4 ? ? ? xxrep 3 ? xxrep 6 ? xxrep 3 ? xxrep 7 ? ? ? xxrep 6 ? 30 xxrep 3 ? 2020 ? ? xxrep 4 ? xxrep 3 ? xxrep 3 ? xxrep 4 ? xxrep 7 ? 3 / 3 \\r\\r\\n @drharshvardhan \\r\\r\\n @narendramodi # indiafightscorona</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fastai_text_classification_data.show_batch(max_n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f1f7cda3",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(fastai_text_classification_data, '..\\classifier_data.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82825602",
   "metadata": {},
   "source": [
    "### Prepare training data for multi-label classification\n",
    "Point to be noted here is that as per our problem statement, ideally we should not make a multi-label classifier, because each tweet can only have one sentiment, but for demonstraion I am making this classifier.\n",
    "In order to make dataloader for multilabel classififation, the target column the the dataset i.e. 'Sentiment' column must have ';' separated multiple values e.g. Neutral;Positive or Negative;Extremely Negative etc. However I will not make any changes to the original data. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f0fc129e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to IPython and Windows limitation, python multiprocessing isn't available now.\n",
      "So `n_workers` has to be changed to 0 to avoid getting stuck\n",
      "Due to IPython and Windows limitation, python multiprocessing isn't available now.\n",
      "So `number_workers` is changed to 0 to avoid getting stuck\n",
      "CPU times: total: 34.3 s\n",
      "Wall time: 34.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "txt_blk = TextBlock.from_df(text_cols = \"OriginalTweet\", seq_len=128)\n",
    "dls_blk = DataBlock(blocks= (txt_blk, MultiCategoryBlock),\n",
    "                   get_x = ColReader(cols = \"text\"), # when we created textblocks from OriginalTweet column, the column name changed to 'text'\n",
    "                   get_y = ColReader(cols = \"Sentiment\", label_delim = \";\")) # Though we do not have any ';' separated value in target, but we assume that if we had something like this\n",
    "\n",
    "fastai_text_classification_multilabel_data= (\n",
    "    dls_blk.dataloaders(\n",
    "        train_data,\n",
    "        valid_pct=0.3,\n",
    "        seed = 12345,\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5bf92c3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>None</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>xxbos xxrep 5 ? ? ? xxrep 7 ? ? ? xxrep 7 ? xxrep 4 ? xxrep 4 ? xxrep 11 ? ? ? xxrep 6 ? xxrep 4 ? , xxrep 3 ? xxrep 3 ? ? ? xxrep 3 ? xxrep 4 ? xxrep 3 ? ? ? ? ? xxrep 4 ? ? ? xxrep 3 ? , xxrep 4 ? ? ? ? ? xxrep 6 ? xxrep 3 ? xxrep 3 ? xxrep 3 ? ? ? xxrep 3 ? \\r\\r\\n▁ xxrep 5 ? xxrep 6 ? ? ? xxrep 3 ? xxrep 4 ? xxrep 4 ? ? ? xxrep 4 ? xxrep 6 ? xxrep 4 ? xxrep 8 ? ? ? xxrep 6 ? ? ? xxrep 5 ? ? ? xxrep 3 ? xxrep 4 ? ? ? xxrep 7 ? xxrep 5 ? - xxrep 8 ? xxrep 5</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>xxbos xxrep 5 ? : xxrep 4 ? xxrep 6 ? ? xxrep 8 ? \\r\\r\\n . \\r\\r\\n ? ? xxrep 6 ? xxrep 4 ? ? ? 500 xxrep 5 ? xxrep 5 ? ? ? xxrep 5 ? xxrep 7 ? xxrep 4 ? xxrep 7 ? ? ? xxrep 3 ? xxrep 7 ? xxrep 4 ? xxrep 6 ? xxrep 9 ? \" xxrep 6 ? \" xxrep 4 ? xxrep 7 ? xxrep 4 ? xxrep 6 ? xxrep 7 ? xxrep 5 ? xxrep 6 ? xxrep 3 ? xxrep 7 ? ? ? xxrep 5 ? .. \\r\\r\\n▁ # xxrep 3 ? _ xxrep 4 ? _ xxrep 6 ? _ xxrep 4 ? _ xxrep 3 ? https : / / t.co / xxunk</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>xxbos # coronavirus # xxmaj kits xxrep 3 ! # xxmaj masks ( 3 ply ) , # handsanitizer ( 75 % ) # xxunk , # xxmaj xxunk , # xxunk . xxmaj order xxmaj now xxrep 3 ! \\r\\r\\n ( # xxmaj families # xxmaj corporations \\r\\r\\n▁ # xxmaj entrepreneurs # hypebeast \\r\\r\\n▁ # xxup ap # xxunk # xxmaj cnn # xxup cdc # newyork # xxmaj atlanta # stlouis # xxmaj denver # xxup xxunk \\r\\r\\n▁ # xxmaj mayors # xxup xxunk ) \\r\\r\\n xxmaj visit : \\r\\r\\n https : / / t.co / xxunk https : / / t.co / xxunk</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fastai_text_classification_multilabel_data.show_batch(max_n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "19464d3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(fastai_text_classification_multilabel_data, '..\\multilabel_classifier_data.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adca0e14",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}