5_model_training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cisVHJmIhbx1"
   },
   "source": [
    "# Text Classification Model\n",
    "\n",
    "In this notebook we will train & explore different Text Classifiers to solve our problem.\n",
    "\n",
    "  1. Train a single classifier variations.\n",
    "  2. Train multiple classifiers on unbalanced dataset.\n",
    "  3. Train multiple classifiers on balanced dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 306
    },
    "colab_type": "code",
    "id": "wWbxHQ8Ji5Be",
    "outputId": "5b6e7cac-9811-4cc0-cc32-4d7bbb92559e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fri Jan 10 07:12:08 2020       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 440.44       Driver Version: 418.67       CUDA Version: 10.1     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
      "| N/A   48C    P8    31W / 149W |      0MiB / 11441MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                       GPU Memory |\n",
      "|  GPU       PID   Type   Process name                             Usage      |\n",
      "|=============================================================================|\n",
      "|  No running processes found                                                 |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "# First let's check what has Google given us ! Thank you Google for the GPU\n",
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "id": "yvrv7VkPi572",
    "outputId": "5a48253b-e916-4f45-d76a-df0b0976e582"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "# Let's mount our G-Drive. Hey !! Because for GPU you now give your data to Google \n",
    "\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "93PiLfdmi7z7",
    "outputId": "97e4cad8-fc04-4b41-b099-bdc63b13279a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tiny-tokenizer\n",
      "  Downloading https://files.pythonhosted.org/packages/8d/0f/aa52c227c5af69914be05723b3deaf221805a4ccbce87643194ef2cdde43/tiny_tokenizer-3.1.0.tar.gz\n",
      "Building wheels for collected packages: tiny-tokenizer\n",
      "  Building wheel for tiny-tokenizer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for tiny-tokenizer: filename=tiny_tokenizer-3.1.0-cp36-none-any.whl size=10550 sha256=d5c57ad3510339b668f5795d2101ffad41fdaae645f2d2c74da2385db55e4b70\n",
      "  Stored in directory: /root/.cache/pip/wheels/d1/c8/36/334497a689fab90128232e86b5829b800dd271a3d5d5959c53\n",
      "Successfully built tiny-tokenizer\n",
      "Installing collected packages: tiny-tokenizer\n",
      "Successfully installed tiny-tokenizer-3.1.0\n",
      "Collecting flair\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/16/22/8fc8e5978ec05b710216735ca47415700e83f304dec7e4281d61cefb6831/flair-0.4.4-py3-none-any.whl (193kB)\n",
      "\u001b[K     |████████████████████████████████| 194kB 6.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: urllib3<1.25,>=1.20 in /usr/local/lib/python3.6/dist-packages (from flair) (1.24.3)\n",
      "Collecting deprecated>=1.2.4\n",
      "  Downloading https://files.pythonhosted.org/packages/f6/89/62912e01f3cede11edcc0abf81298e3439d9c06c8dce644369380ed13f6d/Deprecated-1.2.7-py2.py3-none-any.whl\n",
      "Collecting sqlitedict>=1.6.0\n",
      "  Downloading https://files.pythonhosted.org/packages/0f/1c/c757b93147a219cf1e25cef7e1ad9b595b7f802159493c45ce116521caff/sqlitedict-1.6.0.tar.gz\n",
      "Requirement already satisfied: pytest>=3.6.4 in /usr/local/lib/python3.6/dist-packages (from flair) (3.6.4)\n",
      "Requirement already satisfied: tabulate in /usr/local/lib/python3.6/dist-packages (from flair) (0.8.6)\n",
      "Requirement already satisfied: sklearn in /usr/local/lib/python3.6/dist-packages (from flair) (0.0)\n",
      "Requirement already satisfied: torchvision in /usr/local/lib/python3.6/dist-packages (from flair) (0.4.2)\n",
      "Collecting segtok>=1.5.7\n",
      "  Downloading https://files.pythonhosted.org/packages/1d/59/6ed78856ab99d2da04084b59e7da797972baa0efecb71546b16d48e49d9b/segtok-1.5.7.tar.gz\n",
      "Collecting langdetect\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/59/59/4bc44158a767a6d66de18c4136c8aa90491d56cc951c10b74dd1e13213c9/langdetect-1.0.7.zip (998kB)\n",
      "\u001b[K     |████████████████████████████████| 1.0MB 40.7MB/s \n",
      "\u001b[?25hRequirement already satisfied: gensim>=3.4.0 in /usr/local/lib/python3.6/dist-packages (from flair) (3.6.0)\n",
      "Collecting bpemb>=0.2.9\n",
      "  Downloading https://files.pythonhosted.org/packages/bc/70/468a9652095b370f797ed37ff77e742b11565c6fd79eaeca5f2e50b164a7/bpemb-0.3.0-py3-none-any.whl\n",
      "Requirement already satisfied: torch>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from flair) (1.3.1)\n",
      "Collecting ipython==7.6.1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/2c/c7d44277b599df35af734d8f4142d501192fdb7aef5d04daf882d7eccfbc/ipython-7.6.1-py3-none-any.whl (774kB)\n",
      "\u001b[K     |████████████████████████████████| 778kB 38.3MB/s \n",
      "\u001b[?25hCollecting transformers>=2.0.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/50/10/aeefced99c8a59d828a92cc11d213e2743212d3641c87c82d61b035a7d5c/transformers-2.3.0-py3-none-any.whl (447kB)\n",
      "\u001b[K     |████████████████████████████████| 450kB 43.4MB/s \n",
      "\u001b[?25hRequirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from flair) (2019.12.20)\n",
      "Requirement already satisfied: tiny-tokenizer[all] in /usr/local/lib/python3.6/dist-packages (from flair) (3.1.0)\n",
      "Requirement already satisfied: tqdm>=4.26.0 in /usr/local/lib/python3.6/dist-packages (from flair) (4.28.1)\n",
      "Requirement already satisfied: matplotlib>=2.2.3 in /usr/local/lib/python3.6/dist-packages (from flair) (3.1.2)\n",
      "Requirement already satisfied: ipython-genutils==0.2.0 in /usr/local/lib/python3.6/dist-packages (from flair) (0.2.0)\n",
      "Requirement already satisfied: pymongo in /usr/local/lib/python3.6/dist-packages (from flair) (3.10.0)\n",
      "Collecting mpld3==0.3\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/91/95/a52d3a83d0a29ba0d6898f6727e9858fe7a43f6c2ce81a5fe7e05f0f4912/mpld3-0.3.tar.gz (788kB)\n",
      "\u001b[K     |████████████████████████████████| 798kB 44.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: hyperopt>=0.1.1 in /usr/local/lib/python3.6/dist-packages (from flair) (0.1.2)\n",
      "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.6/dist-packages (from deprecated>=1.2.4->flair) (1.11.2)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (19.3.0)\n",
      "Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (0.7.1)\n",
      "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (1.12.0)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (42.0.2)\n",
      "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (1.8.1)\n",
      "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (1.3.0)\n",
      "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (8.0.2)\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sklearn->flair) (0.22.1)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torchvision->flair) (1.17.5)\n",
      "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision->flair) (6.2.2)\n",
      "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.6/dist-packages (from gensim>=3.4.0->flair) (1.9.0)\n",
      "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from gensim>=3.4.0->flair) (1.4.1)\n",
      "Collecting sentencepiece\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
      "\u001b[K     |████████████████████████████████| 1.0MB 43.6MB/s \n",
      "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from bpemb>=0.2.9->flair) (2.21.0)\n",
      "Collecting prompt-toolkit<2.1.0,>=2.0.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/87/61/2dfea88583d5454e3a64f9308a686071d58d59a55db638268a6413e1eb6d/prompt_toolkit-2.0.10-py3-none-any.whl (340kB)\n",
      "\u001b[K     |████████████████████████████████| 348kB 43.0MB/s \n",
      "\u001b[?25hRequirement already satisfied: pickleshare in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (0.7.5)\n",
      "Requirement already satisfied: backcall in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (0.1.0)\n",
      "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (4.7.0)\n",
      "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (4.3.3)\n",
      "Requirement already satisfied: pygments in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (2.1.3)\n",
      "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (4.4.1)\n",
      "Requirement already satisfied: jedi>=0.10 in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (0.15.2)\n",
      "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->flair) (1.10.47)\n",
      "Collecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/b4/7a41d630547a4afd58143597d5a49e07bfd4c42914d8335b2a5657efc14b/sacremoses-0.0.38.tar.gz (860kB)\n",
      "\u001b[K     |████████████████████████████████| 870kB 38.0MB/s \n",
      "\u001b[?25hCollecting janome; extra == \"all\"\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/79/f0/bd7f90806132d7d9d642d418bdc3e870cfdff5947254ea3cab27480983a7/Janome-0.3.10-py2.py3-none-any.whl (21.5MB)\n",
      "\u001b[K     |████████████████████████████████| 21.5MB 1.7MB/s \n",
      "\u001b[?25hCollecting kytea; extra == \"all\"\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fe/bc/702d01a96d5d094bd9f3c2eb1d12153daf8edf7bf5d78b9a2dae1202df07/kytea-0.1.4-cp36-cp36m-manylinux1_x86_64.whl (3.8MB)\n",
      "\u001b[K     |████████████████████████████████| 3.8MB 38.5MB/s \n",
      "\u001b[?25hCollecting SudachiPy; extra == \"all\"\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/04/c9/40bfb291a7995ad218451ef97083432f998b822e3ecbd9f586f593d2cfb6/SudachiPy-0.4.2-py3-none-any.whl (73kB)\n",
      "\u001b[K     |████████████████████████████████| 81kB 11.3MB/s \n",
      "\u001b[?25hCollecting natto-py; extra == \"all\"\n",
      "  Downloading https://files.pythonhosted.org/packages/f1/14/1d4258247a00b7b8a115563effb1d0bd30501d69580629d36593ce0af92d/natto-py-0.9.2.tar.gz\n",
      "Collecting SudachiDict-core@ https://object-storage.tyo2.conoha.io/v1/nc_2520839e1f9641b08211a5c85243124a/sudachi/SudachiDict_core-20190927.tar.gz ; extra == \"all\"\n",
      "\u001b[?25l  Downloading https://object-storage.tyo2.conoha.io/v1/nc_2520839e1f9641b08211a5c85243124a/sudachi/SudachiDict_core-20190927.tar.gz (70.7MB)\n",
      "\u001b[K     |████████████████████████████████| 70.7MB 40kB/s \n",
      "\u001b[?25hRequirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (0.10.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (1.1.0)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (2.4.6)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (2.6.1)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.6/dist-packages (from hyperopt>=0.1.1->flair) (2.4)\n",
      "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from hyperopt>=0.1.1->flair) (0.16.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sklearn->flair) (0.14.1)\n",
      "Requirement already satisfied: boto>=2.32 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim>=3.4.0->flair) (2.49.0)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->bpemb>=0.2.9->flair) (2.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->bpemb>=0.2.9->flair) (2019.11.28)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->bpemb>=0.2.9->flair) (3.0.4)\n",
      "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.1.0,>=2.0.0->ipython==7.6.1->flair) (0.1.8)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.6/dist-packages (from pexpect; sys_platform != \"win32\"->ipython==7.6.1->flair) (0.6.0)\n",
      "Requirement already satisfied: parso>=0.5.2 in /usr/local/lib/python3.6/dist-packages (from jedi>=0.10->ipython==7.6.1->flair) (0.5.2)\n",
      "Requirement already satisfied: botocore<1.14.0,>=1.13.47 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->flair) (1.13.47)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->flair) (0.9.4)\n",
      "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->flair) (0.2.1)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.0.0->flair) (7.0)\n",
      "Collecting dartsclone~=0.6.0\n",
      "  Downloading https://files.pythonhosted.org/packages/7d/4d/45acbe9d0795d8ceef0fee1f9ac2dcbf27dca3a0578a023fcdc3fef6fd89/dartsclone-0.6.tar.gz\n",
      "Requirement already satisfied: sortedcontainers~=2.1.0 in /usr/local/lib/python3.6/dist-packages (from SudachiPy; extra == \"all\"->tiny-tokenizer[all]->flair) (2.1.0)\n",
      "Requirement already satisfied: cffi in /usr/local/lib/python3.6/dist-packages (from natto-py; extra == \"all\"->tiny-tokenizer[all]->flair) (1.13.2)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.47->boto3->transformers>=2.0.0->flair) (0.15.2)\n",
      "Requirement already satisfied: Cython in /usr/local/lib/python3.6/dist-packages (from dartsclone~=0.6.0->SudachiPy; extra == \"all\"->tiny-tokenizer[all]->flair) (0.29.14)\n",
      "Requirement already satisfied: pycparser in /usr/local/lib/python3.6/dist-packages (from cffi->natto-py; extra == \"all\"->tiny-tokenizer[all]->flair) (2.19)\n",
      "Building wheels for collected packages: sqlitedict, segtok, langdetect, mpld3, sacremoses, natto-py, SudachiDict-core, dartsclone\n",
      "  Building wheel for sqlitedict (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sqlitedict: filename=sqlitedict-1.6.0-cp36-none-any.whl size=14689 sha256=4b42bff00f316178d43f662a9a72665bfc0cfcee126c562759c641bed98fc005\n",
      "  Stored in directory: /root/.cache/pip/wheels/bd/57/d3/907c3ee02d35e66f674ad0106e61f06eeeb98f6ee66a6cc3fe\n",
      "  Building wheel for segtok (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for segtok: filename=segtok-1.5.7-cp36-none-any.whl size=23258 sha256=d45fe01bd8dbba687ab4b8e12f3bd6d8dcd344999b77842ab42fc8ba8132268e\n",
      "  Stored in directory: /root/.cache/pip/wheels/15/ee/a8/6112173f1386d33eebedb3f73429cfa41a4c3084556bcee254\n",
      "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for langdetect: filename=langdetect-1.0.7-cp36-none-any.whl size=993460 sha256=7a15a2d57069eadc74a74d98e20c8fa18bfc0c9057e5a755ebc3527b6eee31c5\n",
      "  Stored in directory: /root/.cache/pip/wheels/ec/0c/a9/1647275e7ef5014e7b83ff30105180e332867d65e7617ddafe\n",
      "  Building wheel for mpld3 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for mpld3: filename=mpld3-0.3-cp36-none-any.whl size=116679 sha256=903a104fe5d520c425d52ba4f05db3e4dbce7e777018ca278d18a3e91eb5617b\n",
      "  Stored in directory: /root/.cache/pip/wheels/c0/47/fb/8a64f89aecfe0059830479308ad42d62e898a3e3cefdf6ba28\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.38-cp36-none-any.whl size=884629 sha256=9f3b80b89ae91665ca0c8fbd1965d135449fd9ab3a21c1a3f0ac05e440e05003\n",
      "  Stored in directory: /root/.cache/pip/wheels/6d/ec/1a/21b8912e35e02741306f35f66c785f3afe94de754a0eaf1422\n",
      "  Building wheel for natto-py (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for natto-py: filename=natto_py-0.9.2-cp36-none-any.whl size=45164 sha256=737f3176aa7805d9e162780cb740d941d809e21b86282754e3677a1aba7054bc\n",
      "  Stored in directory: /root/.cache/pip/wheels/ce/51/dd/67f87608b124a23eecf5c1fc3557cc0b7ffdeae33fe6ee89df\n",
      "  Building wheel for SudachiDict-core (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for SudachiDict-core: filename=SudachiDict_core-20190927-cp36-none-any.whl size=70878518 sha256=909397579f947909ab0b55dee0f7eb4da532a7092e3ff93a2b6cc887113c7b4f\n",
      "  Stored in directory: /root/.cache/pip/wheels/22/d8/6e/b107d7fef6e80915aa1e46db741b98a3da011f567526347ccc\n",
      "  Building wheel for dartsclone (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for dartsclone: filename=dartsclone-0.6-cp36-cp36m-linux_x86_64.whl size=413249 sha256=aaf67253aea70b37c131e19ec491143f6c6d34dc9ed7ae07d64eefbec0548157\n",
      "  Stored in directory: /root/.cache/pip/wheels/be/cd/70/fe43307bf7398243155108f4f5a258ef336923d65ec4af93cd\n",
      "Successfully built sqlitedict segtok langdetect mpld3 sacremoses natto-py SudachiDict-core dartsclone\n",
      "\u001b[31mERROR: jupyter-console 5.2.0 has requirement prompt-toolkit<2.0.0,>=1.0.0, but you'll have prompt-toolkit 2.0.10 which is incompatible.\u001b[0m\n",
      "\u001b[31mERROR: google-colab 1.0.0 has requirement ipython~=5.5.0, but you'll have ipython 7.6.1 which is incompatible.\u001b[0m\n",
      "Installing collected packages: deprecated, sqlitedict, segtok, langdetect, sentencepiece, bpemb, prompt-toolkit, ipython, sacremoses, transformers, mpld3, flair, janome, kytea, dartsclone, SudachiPy, natto-py, SudachiDict-core\n",
      "  Found existing installation: prompt-toolkit 1.0.18\n",
      "    Uninstalling prompt-toolkit-1.0.18:\n",
      "      Successfully uninstalled prompt-toolkit-1.0.18\n",
      "  Found existing installation: ipython 5.5.0\n",
      "    Uninstalling ipython-5.5.0:\n",
      "      Successfully uninstalled ipython-5.5.0\n",
      "Successfully installed SudachiDict-core-20190927 SudachiPy-0.4.2 bpemb-0.3.0 dartsclone-0.6 deprecated-1.2.7 flair-0.4.4 ipython-7.6.1 janome-0.3.10 kytea-0.1.4 langdetect-1.0.7 mpld3-0.3 natto-py-0.9.2 prompt-toolkit-2.0.10 sacremoses-0.0.38 segtok-1.5.7 sentencepiece-0.1.85 sqlitedict-1.6.0 transformers-2.3.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.colab-display-data+json": {
       "pip_warning": {
        "packages": [
         "IPython",
         "prompt_toolkit"
        ]
       }
      }
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Install necessary packages and restart the environment\n",
    "\n",
    "! pip install tiny-tokenizer\n",
    "! pip install  flair\n",
    "# ! pip install -U tensorflow-gpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 63
    },
    "colab_type": "code",
    "id": "16x7OOFJjIJX",
    "outputId": "703957c8-1236-4847-d949-f6f59cc86ca0"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p style=\"color: red;\">\n",
       "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
       "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
       "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
       "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Let's import our packages !\n",
    "\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import html\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "from sklearn.model_selection import train_test_split\n",
    "import flair\n",
    "import pickle\n",
    "from torch.optim.adam import Adam\n",
    "\n",
    "from flair.data import Corpus\n",
    "from flair.datasets import ClassificationCorpus\n",
    "from flair.embeddings import WordEmbeddings, FlairEmbeddings, DocumentRNNEmbeddings,FastTextEmbeddings\n",
    "from flair.models import TextClassifier\n",
    "from flair.trainers import ModelTrainer\n",
    "from flair.samplers import ImbalancedClassificationDatasetSampler\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zKwcewlCjSsy"
   },
   "outputs": [],
   "source": [
    "# Load the TensorBoard notebook extension\n",
    "%load_ext tensorboard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ItWgf2pyjwQy"
   },
   "source": [
    "# 1. Train a Single Classifier\n",
    "\n",
    "The first approach, would be to load the entire dataset(~1M) and train a single classifier powerful & mighty enough to perform great.\n",
    "\n",
    "Challenges :\n",
    "  * Super slow to experiment with different architecture\n",
    "  * Super slow to train a model\n",
    "  * Super slow for Hyper-parameter tuning\n",
    "  * Highly Skewed Dataset\n",
    "\n",
    "\n",
    "Experiments :\n",
    "  * Try stacking multiple embeddings at #3\n",
    "  * Try custom embeddings at #3\n",
    "  * Try different RNN cell at #4\n",
    "  * Try different hidden size at #4\n",
    "  * All hyperparameters are tunable for extensive experimenting at #7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-2aapmebk03R"
   },
   "outputs": [],
   "source": [
    "path = '/content/drive/My Drive/ICDMAI_Tutorial/notebook/'\n",
    "corpus_path = path+'training_data/classification_corpus.pkl'\n",
    "label_dict_path = path+'training_data/classification_corpus_label_dict.pkl'\n",
    "\n",
    "\n",
    "# 1. Reading Corpus File : which we prepared before-hand\n",
    "with open(corpus_path, mode='rb') as f:\n",
    "  corpus = pickle.load(f)\n",
    "\n",
    "# 2. Reading Corpus Dictionary : which we computed & saved\n",
    "with open(label_dict_path, mode='rb') as f:\n",
    "  label_dict = pickle.load(f)\n",
    "\n",
    "# 3. make a list of word embeddings \n",
    "word_embeddings = [\n",
    "  WordEmbeddings('glove'),\n",
    "\n",
    "  # comment in flair embeddings for state-of-the-art results\n",
    "  # FlairEmbeddings('news-forward'),\n",
    "  # FlairEmbeddings('news-backward'),\n",
    "]\n",
    "\n",
    "# 4. initialize document embedding by passing list of word embeddings\n",
    "## Can choose between many RNN types (GRU by default, to change use rnn_type parameter)\n",
    "document_embeddings = DocumentRNNEmbeddings(\n",
    "  word_embeddings,\n",
    "  hidden_size=128,\n",
    "  reproject_words=True,\n",
    "  reproject_words_dimension=256,\n",
    ")\n",
    "\n",
    "# 5. create the text classifier\n",
    "classifier = TextClassifier(document_embeddings,\n",
    "                            label_dictionary=label_dict,\n",
    "                            multi_label=True)\n",
    "\n",
    "# 6. initialize the text classifier trainer\n",
    "trainer = ModelTrainer(classifier,\n",
    "                        corpus,\n",
    "                        optimizer=Adam,\n",
    "                        use_tensorboard=True)\n",
    "\n",
    "# 7. start the training\n",
    "model_path = path + '/model/full_model'\n",
    "trainer.train(model_path,\n",
    "              learning_rate=0.06,\n",
    "              mini_batch_size=32,\n",
    "              anneal_factor=0.5,\n",
    "              patience=5,\n",
    "              max_epochs=2,\n",
    "              checkpoint=True,\n",
    "              embeddings_storage_mode='gpu',\n",
    "              num_workers=12)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "itvLVvPC2VB3"
   },
   "source": [
    "# 2. Train Multiple Classifiers on unbalanced Dataset\n",
    "\n",
    "A single classifier fails to capture and fit the data, performing poorly on all the metrices. This led us to divide the problem into sub-groups and FOCUS more on individual group than Training a Titan Model for everything.\n",
    "\n",
    "In this section, we would train multiple classifiers on the different groups that we create.\n",
    "\n",
    "### Experiments \n",
    "\n",
    "  1. We chose a couple of representative groups\n",
    "  2. We try different architectures & embeddings\n",
    "  3. We train only 2 epochs for quick results\n",
    "  4. We manually try couple of Hyper-parameter tuning based on our Hypothesis\n",
    "\n",
    "\n",
    "### Things to try & Build Hypothesis on:\n",
    "  1. GRU/LSTM cells\n",
    "  2. Number of RNN Layers \n",
    "  3. Hidden Units / Time steps / Sequence Length\n",
    "  4. Embeddings\n",
    "  5. Batch Size\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "oHuiL2795lE8",
    "outputId": "dbd2ae22-2fa7-41af-bb2a-f2763d727cf2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================================\n",
      "Group ID : 10\n",
      "================================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:402: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
      "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-01-10 07:16:18,761 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:16:18,767 Model: \"TextClassifier(\n",
      "  (document_embeddings): DocumentRNNEmbeddings(\n",
      "    (embeddings): StackedEmbeddings(\n",
      "      (list_embedding_0): WordEmbeddings('/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/word_embedding/gensim_model')\n",
      "    )\n",
      "    (word_reprojection_map): Linear(in_features=300, out_features=256, bias=True)\n",
      "    (rnn): GRU(256, 256, num_layers=2, batch_first=True)\n",
      "    (dropout): Dropout(p=0.5, inplace=False)\n",
      "  )\n",
      "  (decoder): Linear(in_features=256, out_features=4, bias=True)\n",
      "  (loss_function): BCEWithLogitsLoss()\n",
      ")\"\n",
      "2020-01-10 07:16:18,771 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:16:18,775 Corpus: \"Corpus: 9085 train + 1947 dev + 1947 test sentences\"\n",
      "2020-01-10 07:16:18,779 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:16:18,781 Parameters:\n",
      "2020-01-10 07:16:18,783  - learning_rate: \"0.1\"\n",
      "2020-01-10 07:16:18,785  - mini_batch_size: \"128\"\n",
      "2020-01-10 07:16:18,787  - patience: \"5\"\n",
      "2020-01-10 07:16:18,790  - anneal_factor: \"0.5\"\n",
      "2020-01-10 07:16:18,792  - max_epochs: \"2\"\n",
      "2020-01-10 07:16:18,795  - shuffle: \"True\"\n",
      "2020-01-10 07:16:18,799  - train_with_dev: \"False\"\n",
      "2020-01-10 07:16:18,801  - batch_growth_annealing: \"False\"\n",
      "2020-01-10 07:16:18,803 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:16:18,805 Model training base path: \"/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/standard/group/10\"\n",
      "2020-01-10 07:16:18,808 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:16:18,810 Device: cuda:0\n",
      "2020-01-10 07:16:18,812 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:16:18,814 Embeddings storage mode: cpu\n",
      "2020-01-10 07:16:18,822 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:16:27,355 epoch 1 - iter 0/71 - loss 0.69800305 - samples/sec: 234.93\n",
      "2020-01-10 07:16:54,317 epoch 1 - iter 7/71 - loss 1.66936630 - samples/sec: 37.20\n",
      "2020-01-10 07:17:17,561 epoch 1 - iter 14/71 - loss 1.38349822 - samples/sec: 42.49\n",
      "2020-01-10 07:17:38,408 epoch 1 - iter 21/71 - loss 1.24392257 - samples/sec: 47.27\n",
      "2020-01-10 07:17:59,056 epoch 1 - iter 28/71 - loss 1.16605080 - samples/sec: 46.40\n",
      "2020-01-10 07:18:19,787 epoch 1 - iter 35/71 - loss 1.09455537 - samples/sec: 50.08\n",
      "2020-01-10 07:18:42,121 epoch 1 - iter 42/71 - loss 1.03435152 - samples/sec: 49.59\n",
      "2020-01-10 07:19:03,394 epoch 1 - iter 49/71 - loss 0.98697994 - samples/sec: 47.43\n",
      "2020-01-10 07:19:23,693 epoch 1 - iter 56/71 - loss 0.94596896 - samples/sec: 50.40\n",
      "2020-01-10 07:19:44,678 epoch 1 - iter 63/71 - loss 0.91304241 - samples/sec: 47.70\n",
      "2020-01-10 07:20:03,609 epoch 1 - iter 70/71 - loss 0.88929346 - samples/sec: 51.81\n",
      "2020-01-10 07:20:04,544 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:20:04,549 EPOCH 1 done: loss 0.8893 - lr 0.1000\n",
      "2020-01-10 07:20:43,607 DEV : loss 0.600093424320221 - score 0.0028\n",
      "2020-01-10 07:20:52,443 BAD EPOCHS (no improvement): 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type TextClassifier. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type DocumentRNNEmbeddings. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type StackedEmbeddings. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type WordEmbeddings. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type Linear. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type GRU. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type Dropout. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type BCEWithLogitsLoss. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-01-10 07:21:27,452 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:21:35,644 epoch 2 - iter 0/71 - loss 0.68568194 - samples/sec: 287.36\n",
      "2020-01-10 07:22:05,410 epoch 2 - iter 7/71 - loss 0.67302275 - samples/sec: 38.41\n",
      "2020-01-10 07:22:29,020 epoch 2 - iter 14/71 - loss 0.70303029 - samples/sec: 43.61\n",
      "2020-01-10 07:22:52,829 epoch 2 - iter 21/71 - loss 0.77519718 - samples/sec: 42.53\n",
      "2020-01-10 07:23:12,962 epoch 2 - iter 28/71 - loss 0.81017300 - samples/sec: 52.21\n",
      "2020-01-10 07:23:33,755 epoch 2 - iter 35/71 - loss 0.81906394 - samples/sec: 47.67\n",
      "2020-01-10 07:23:54,393 epoch 2 - iter 42/71 - loss 0.80896194 - samples/sec: 48.46\n",
      "2020-01-10 07:24:13,609 epoch 2 - iter 49/71 - loss 0.81108198 - samples/sec: 52.12\n",
      "2020-01-10 07:24:34,933 epoch 2 - iter 56/71 - loss 0.81042614 - samples/sec: 50.62\n",
      "2020-01-10 07:24:56,373 epoch 2 - iter 63/71 - loss 0.80587265 - samples/sec: 48.62\n",
      "2020-01-10 07:25:16,368 epoch 2 - iter 70/71 - loss 0.80090538 - samples/sec: 49.21\n",
      "2020-01-10 07:25:17,438 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:25:17,442 EPOCH 2 done: loss 0.8009 - lr 0.1000\n",
      "2020-01-10 07:25:54,297 DEV : loss 0.6496628522872925 - score 0.4533\n",
      "2020-01-10 07:26:02,807 BAD EPOCHS (no improvement): 0\n",
      "2020-01-10 07:27:06,264 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:27:06,272 Testing using best model ...\n",
      "2020-01-10 07:27:06,283 loading file /content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/standard/group/10/best-model.pt\n",
      "2020-01-10 07:28:15,099 0.3776\t0.592\t0.4611\n",
      "2020-01-10 07:28:15,105 \n",
      "MICRO_AVG: acc 0.2997 - f1-score 0.4611\n",
      "MACRO_AVG: acc 0.2048 - f1-score 0.311525\n",
      "apache-spark tp: 10 - fp: 12 - fn: 275 - tn: 1650 - precision: 0.4545 - recall: 0.0351 - accuracy: 0.0337 - f1-score: 0.0652\n",
      "hadoop     tp: 41 - fp: 14 - fn: 444 - tn: 1448 - precision: 0.7455 - recall: 0.0845 - accuracy: 0.0822 - f1-score: 0.1518\n",
      "haskell    tp: 410 - fp: 1100 - fn: 23 - tn: 414 - precision: 0.2715 - recall: 0.9469 - accuracy: 0.2674 - f1-score: 0.4220\n",
      "scala      tp: 758 - fp: 883 - fn: 98 - tn: 208 - precision: 0.4619 - recall: 0.8855 - accuracy: 0.4359 - f1-score: 0.6071\n",
      "2020-01-10 07:28:15,109 ----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "for grp_id in [10] :\n",
    "  print(\"================================================================================================\")\n",
    "  print(\"Group ID : {}\".format(grp_id))\n",
    "  print(\"================================================================================================\")\n",
    "\n",
    "  path = '/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/standard/group/'\n",
    "  base_path = '/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/standard/group/' + str(grp_id) + '/'\n",
    "\n",
    "  corpus_path =  path + str(grp_id) + '/classification_corpus.pkl'\n",
    "  label_dict_path = path + str(grp_id) + '/classification_corpus_label_dict.pkl'\n",
    "\n",
    "  # 1. Reading Corpus File : which we prepared before-hand\n",
    "  with open(corpus_path, mode='rb') as f:\n",
    "    corpus = pickle.load(f)\n",
    "\n",
    "  # 2. Reading Corpus Dictionary : which we computed & saved\n",
    "  with open(label_dict_path, mode='rb') as f:\n",
    "    label_dict = pickle.load(f)\n",
    "\n",
    "  # 3. make a list of word embeddings \n",
    "  word_embeddings = [ \n",
    "                     WordEmbeddings('/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/word_embedding/gensim_model'), # Custom Word Embedding \n",
    "                    ## comment in different embeddings for state-of-the-art results\n",
    "                     \n",
    "                    #  WordEmbeddings('glove'),                 \n",
    "                    # FlairEmbeddings('news-forward'),\n",
    "                    # FlairEmbeddings('news-backward'),\n",
    "                     \n",
    "    ]\n",
    "\n",
    "  # Can choose between many RNN types (GRU by default, to change use rnn_type parameter)\n",
    "  document_embeddings = DocumentRNNEmbeddings(\n",
    "    word_embeddings,\n",
    "    hidden_size=256, # Build a hypothesis for different values\n",
    "    rnn_layers = 2,  # Build a hypothesis for different values\n",
    "    reproject_words=True, \n",
    "    reproject_words_dimension=256\n",
    "  )\n",
    "\n",
    "  classifier = TextClassifier(document_embeddings,\n",
    "                            label_dictionary=label_dict,\n",
    "                            multi_label=True)\n",
    "\n",
    "  # 6. initialize the text classifier trainer\n",
    "  trainer = ModelTrainer(classifier,\n",
    "                          corpus,\n",
    "                          optimizer=Adam\n",
    "                          )\n",
    "\n",
    "  # model_path = path + '/model_70_30/full_model_1'\n",
    "\n",
    "  trainer.train(base_path,\n",
    "                learning_rate=0.1,\n",
    "                mini_batch_size=128,\n",
    "                anneal_factor=0.5,\n",
    "                patience=5,\n",
    "                max_epochs=2,\n",
    "                checkpoint=True,\n",
    "                embeddings_storage_mode='gpu'\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LHNjIkkdQKp0"
   },
   "source": [
    "Once with you experiments, you have finalised top 2-3 architecture and configurations, then use it to train the classifiers.\n",
    "\n",
    "**Note** : You should individually run these experiments for all the groups and train custom model for each of them. Here we will be using a vanila configuration for all the groups."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cbMcPewzQjrS"
   },
   "outputs": [],
   "source": [
    "for grp_id in range(1,15) :\n",
    "\n",
    "  print(\"================================================================================================\")\n",
    "  print(\"Group ID : {}\".format(grp_id))\n",
    "  print(\"================================================================================================\")\n",
    "\n",
    "  path = '/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/training_data/standard/group/'\n",
    "  base_path = '/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/training_data/standard/group/' + str(grp_id) + '/'\n",
    "\n",
    "  corpus_path =  path + str(grp_id) + '/classification_corpus.pkl'\n",
    "  label_dict_path = path + str(grp_id) + '/classification_corpus_label_dict.pkl'\n",
    "\n",
    "  # 1. Reading Corpus File : which we prepared before-hand\n",
    "  with open(corpus_path, mode='rb') as f:\n",
    "    corpus = pickle.load(f)\n",
    "\n",
    "  # 2. Reading Corpus Dictionary : which we computed & saved\n",
    "  with open(label_dict_path, mode='rb') as f:\n",
    "    label_dict = pickle.load(f)\n",
    "\n",
    "  # 3. make a list of word embeddings \n",
    "  word_embeddings = [ \n",
    "                     WordEmbeddings('/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/word_embedding/gensim_model'), # Custom Word Embedding \n",
    "                     \n",
    "    ]\n",
    "\n",
    "  # Can choose between many RNN types (GRU by default, to change use rnn_type parameter)\n",
    "  document_embeddings = DocumentRNNEmbeddings(\n",
    "    word_embeddings,\n",
    "    hidden_size=128, \n",
    "    rnn_layers = 1,  \n",
    "    reproject_words=True, \n",
    "    reproject_words_dimension=256\n",
    "  )\n",
    "\n",
    "  classifier = TextClassifier(document_embeddings,\n",
    "                            label_dictionary=label_dict,\n",
    "                            multi_label=True)\n",
    "\n",
    "  # 6. initialize the text classifier trainer\n",
    "  trainer = ModelTrainer(classifier,\n",
    "                          corpus,\n",
    "                          optimizer=Adam\n",
    "                          )\n",
    "\n",
    "\n",
    "  trainer.train(base_path,\n",
    "                learning_rate=0.1,\n",
    "                mini_batch_size=128,\n",
    "                anneal_factor=0.5,\n",
    "                patience=5,\n",
    "                max_epochs=10,\n",
    "                checkpoint=True,\n",
    "                embeddings_storage_mode='gpu'\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mmbE8U1yWhSM"
   },
   "source": [
    "# 3. Train Multiple Classifiers on Balanced Dataset\n",
    "\n",
    "\n",
    "Though now we see that there has been some improvement in the metrices with multiple classifiers in their respective groups validation/test datasets. it should be still be noted that those groups are fairly skewed and gives a hard time to the model to learn meaningful relations.\n",
    "\n",
    "Hence now we train the model with the normalised dataset that we prepared. \n",
    "\n",
    "**Note** : It should be noted that for every dataset/problem different ways of balancing/normalising the dataset works. Here clipping off worked well for us but penalising loss for less representative class did not.\n",
    "\n",
    "We run the same set of experiments & build a hypothesis.\n",
    "\n",
    "### Experiments \n",
    "\n",
    "  1. We chose a couple of representative groups\n",
    "  2. We try different architectures & embeddings\n",
    "  3. We train only 2 epochs for quick results\n",
    "  4. We manually try couple of Hyper-parameter tuning based on our Hypothesis\n",
    "\n",
    "\n",
    "### Things to try & Build Hypothesis on:\n",
    "  1. GRU/LSTM cells\n",
    "  2. Number of RNN Layers \n",
    "  3. Hidden Units / Time steps / Sequence Length\n",
    "  4. Embeddings\n",
    "  5. Batch Size\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "kkMxNCwSY60t",
    "outputId": "d546a866-f1dd-451a-b481-0f2e243f71c0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================================\n",
      "Group ID : 10\n",
      "================================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:402: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
      "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-01-10 07:34:12,540 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:34:12,546 Model: \"TextClassifier(\n",
      "  (document_embeddings): DocumentRNNEmbeddings(\n",
      "    (embeddings): StackedEmbeddings(\n",
      "      (list_embedding_0): WordEmbeddings('/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/word_embedding/gensim_model')\n",
      "    )\n",
      "    (word_reprojection_map): Linear(in_features=300, out_features=256, bias=True)\n",
      "    (rnn): LSTM(256, 64, batch_first=True, bidirectional=True)\n",
      "  )\n",
      "  (decoder): Linear(in_features=256, out_features=4, bias=True)\n",
      "  (loss_function): BCEWithLogitsLoss()\n",
      ")\"\n",
      "2020-01-10 07:34:12,550 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:34:12,553 Corpus: \"Corpus: 8811 train + 1101 dev + 1102 test sentences\"\n",
      "2020-01-10 07:34:12,556 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:34:12,558 Parameters:\n",
      "2020-01-10 07:34:12,562  - learning_rate: \"0.06\"\n",
      "2020-01-10 07:34:12,563  - mini_batch_size: \"64\"\n",
      "2020-01-10 07:34:12,567  - patience: \"5\"\n",
      "2020-01-10 07:34:12,569  - anneal_factor: \"0.5\"\n",
      "2020-01-10 07:34:12,570  - max_epochs: \"2\"\n",
      "2020-01-10 07:34:12,571  - shuffle: \"True\"\n",
      "2020-01-10 07:34:12,573  - train_with_dev: \"False\"\n",
      "2020-01-10 07:34:12,575  - batch_growth_annealing: \"False\"\n",
      "2020-01-10 07:34:12,576 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:34:12,583 Model training base path: \"/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/normalised/group/10\"\n",
      "2020-01-10 07:34:12,589 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:34:12,592 Device: cuda:0\n",
      "2020-01-10 07:34:12,600 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:34:12,601 Embeddings storage mode: cpu\n",
      "2020-01-10 07:34:56,964 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:35:00,629 epoch 1 - iter 0/138 - loss 0.71194822 - samples/sec: 396.06\n",
      "2020-01-10 07:35:19,601 epoch 1 - iter 13/138 - loss 0.79409514 - samples/sec: 53.58\n",
      "2020-01-10 07:35:38,517 epoch 1 - iter 26/138 - loss 0.70222513 - samples/sec: 49.06\n",
      "2020-01-10 07:35:56,932 epoch 1 - iter 39/138 - loss 0.66638391 - samples/sec: 51.28\n",
      "2020-01-10 07:36:14,703 epoch 1 - iter 52/138 - loss 0.65179868 - samples/sec: 52.45\n",
      "2020-01-10 07:36:32,276 epoch 1 - iter 65/138 - loss 0.64149286 - samples/sec: 54.07\n",
      "2020-01-10 07:36:53,336 epoch 1 - iter 78/138 - loss 0.63322508 - samples/sec: 46.78\n",
      "2020-01-10 07:37:10,162 epoch 1 - iter 91/138 - loss 0.62800710 - samples/sec: 54.04\n",
      "2020-01-10 07:37:29,944 epoch 1 - iter 104/138 - loss 0.62297002 - samples/sec: 49.33\n",
      "2020-01-10 07:37:48,064 epoch 1 - iter 117/138 - loss 0.62282760 - samples/sec: 54.36\n",
      "2020-01-10 07:38:05,025 epoch 1 - iter 130/138 - loss 0.62036079 - samples/sec: 59.15\n",
      "2020-01-10 07:38:13,744 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:38:13,746 EPOCH 1 done: loss 0.6204 - lr 0.0600\n",
      "2020-01-10 07:38:36,435 DEV : loss 0.6815922856330872 - score 0.3479\n",
      "2020-01-10 07:38:41,600 BAD EPOCHS (no improvement): 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type TextClassifier. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type DocumentRNNEmbeddings. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type StackedEmbeddings. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type WordEmbeddings. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type Linear. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type LSTM. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type BCEWithLogitsLoss. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-01-10 07:39:12,944 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:39:22,359 epoch 2 - iter 0/138 - loss 0.68621749 - samples/sec: 374.58\n",
      "2020-01-10 07:39:47,127 epoch 2 - iter 13/138 - loss 0.61067164 - samples/sec: 41.58\n",
      "2020-01-10 07:40:05,630 epoch 2 - iter 26/138 - loss 0.61288287 - samples/sec: 48.34\n",
      "2020-01-10 07:40:23,371 epoch 2 - iter 39/138 - loss 0.61177309 - samples/sec: 51.07\n",
      "2020-01-10 07:40:40,518 epoch 2 - iter 52/138 - loss 0.61445505 - samples/sec: 52.62\n",
      "2020-01-10 07:40:58,839 epoch 2 - iter 65/138 - loss 0.62030486 - samples/sec: 49.07\n",
      "2020-01-10 07:41:18,826 epoch 2 - iter 78/138 - loss 0.62219412 - samples/sec: 45.22\n",
      "2020-01-10 07:41:37,242 epoch 2 - iter 91/138 - loss 0.61707596 - samples/sec: 53.72\n",
      "2020-01-10 07:41:55,451 epoch 2 - iter 104/138 - loss 0.61539865 - samples/sec: 56.66\n",
      "2020-01-10 07:42:14,409 epoch 2 - iter 117/138 - loss 0.61404586 - samples/sec: 51.61\n",
      "2020-01-10 07:42:30,974 epoch 2 - iter 130/138 - loss 0.61217673 - samples/sec: 60.44\n",
      "2020-01-10 07:42:39,588 ----------------------------------------------------------------------------------------------------\n",
      "2020-01-10 07:42:39,590 EPOCH 2 done: loss 0.6113 - lr 0.0600\n",
      "2020-01-10 07:43:02,585 DEV : loss 0.5858250260353088 - score 0.4176\n",
      "2020-01-10 07:43:06,987 BAD EPOCHS (no improvement): 0\n"
     ]
    }
   ],
   "source": [
    "for grp_id in [10] :\n",
    "  print(\"================================================================================================\")\n",
    "  print(\"Group ID : {}\".format(grp_id))\n",
    "  print(\"================================================================================================\")\n",
    "\n",
    "  path = '/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/normalised/group/'\n",
    "  base_path = '/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/normalised/group/' + str(grp_id) + '/'\n",
    "\n",
    "  corpus_path =  path + str(grp_id) + '/classification_corpus.pkl'\n",
    "  label_dict_path = path + str(grp_id) + '/classification_corpus_label_dict.pkl'\n",
    "\n",
    "  # 1. Reading Corpus File : which we prepared before-hand\n",
    "  with open(corpus_path, mode='rb') as f:\n",
    "    corpus = pickle.load(f)\n",
    "\n",
    "  # 2. Reading Corpus Dictionary : which we computed & saved\n",
    "  with open(label_dict_path, mode='rb') as f:\n",
    "    label_dict = pickle.load(f)\n",
    "\n",
    "  # 3. make a list of word embeddings \n",
    "  word_embeddings = [ \n",
    "                     WordEmbeddings('/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/word_embedding/gensim_model'), # Custom Word Embedding \n",
    "                    ## comment in different embeddings for state-of-the-art results\n",
    "                     \n",
    "                    #  WordEmbeddings('glove'),                 \n",
    "                    # FlairEmbeddings('news-forward'),\n",
    "                    # FlairEmbeddings('news-backward'),\n",
    "                     \n",
    "    ]\n",
    "\n",
    "  # Can choose between many RNN types (GRU by default, to change use rnn_type parameter)\n",
    "  document_embeddings = DocumentRNNEmbeddings(\n",
    "    word_embeddings,\n",
    "    hidden_size=64, # Build a hypothesis for different values\n",
    "    rnn_layers = 1,  # Build a hypothesis for different values\n",
    "    bidirectional = True, # Trying changing the behaviour of the model\n",
    "    reproject_words=True, \n",
    "    reproject_words_dimension=256,\n",
    "    dropout = 0 ,\n",
    "    rnn_type = 'LSTM'\n",
    "  )\n",
    "\n",
    "  classifier = TextClassifier(document_embeddings,\n",
    "                            label_dictionary=label_dict,\n",
    "                            multi_label_threshold = 0.3 , # Check with different Thresholds\n",
    "                            multi_label=True)\n",
    "\n",
    "  # 6. initialize the text classifier trainer\n",
    "  trainer = ModelTrainer(classifier,\n",
    "                          corpus,\n",
    "                          optimizer=Adam\n",
    "                          )\n",
    "\n",
    "\n",
    "  trainer.train(base_path,\n",
    "                learning_rate=0.06,\n",
    "                mini_batch_size=64,\n",
    "                anneal_factor=0.5,\n",
    "                patience=5,\n",
    "                max_epochs=2,\n",
    "                checkpoint=True,\n",
    "                embeddings_storage_mode='gpu',\n",
    "#                 sampler=ImbalancedClassificationDatasetSampler # Check if puishing the mis-classification of less frequent labels heavily helps?\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-5CX_0uJaGAN"
   },
   "source": [
    "Once with you experiments, you have finalised top 2-3 architecture and configurations, then use it to train the classifiers.\n",
    "\n",
    "**Note** : You should individually run these experiments for all the groups and train custom model for each of them. Here we will be using a vanila configuration for all the groups.\n",
    "\n",
    "**Fun Fact** : We ran ~80 experiments for architecture for this small dataset itself to build the hypothesis for this Demonstrations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bJhCZEYjaTyR"
   },
   "outputs": [],
   "source": [
    "for grp_id in range(1,15) :\n",
    "  print(\"================================================================================================\")\n",
    "  print(\"Group ID : {}\".format(grp_id))\n",
    "  print(\"================================================================================================\")\n",
    "\n",
    "  path = '/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/normalised/group/'\n",
    "  base_path = '/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/normalised/group/' + str(grp_id) + '/'\n",
    "\n",
    "  corpus_path =  path + str(grp_id) + '/classification_corpus.pkl'\n",
    "  label_dict_path = path + str(grp_id) + '/classification_corpus_label_dict.pkl'\n",
    "\n",
    "  # 1. Reading Corpus File : which we prepared before-hand\n",
    "  with open(corpus_path, mode='rb') as f:\n",
    "    corpus = pickle.load(f)\n",
    "\n",
    "  # 2. Reading Corpus Dictionary : which we computed & saved\n",
    "  with open(label_dict_path, mode='rb') as f:\n",
    "    label_dict = pickle.load(f)\n",
    "\n",
    "  # 3. make a list of word embeddings \n",
    "  word_embeddings = [ \n",
    "                     WordEmbeddings('/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/word_embedding/gensim_model'), # Custom Word Embedding \n",
    "                     \n",
    "    ]\n",
    "\n",
    "  # Can choose between many RNN types (GRU by default, to change use rnn_type parameter)\n",
    "  document_embeddings = DocumentRNNEmbeddings(\n",
    "    word_embeddings,\n",
    "    hidden_size=64, # Build a hypothesis for different values\n",
    "    rnn_layers = 1,  # Build a hypothesis for different values\n",
    "    bidirectional = True, # Trying changing the behaviour of the model\n",
    "    reproject_words=True, \n",
    "    reproject_words_dimension=256,\n",
    "    dropout = 0 ,\n",
    "    rnn_type = 'LSTM'\n",
    "  )\n",
    "\n",
    "  classifier = TextClassifier(document_embeddings,\n",
    "                            label_dictionary=label_dict,\n",
    "                            multi_label_threshold = 0.1 , # Check with different Thresholds\n",
    "                            multi_label=True)\n",
    "\n",
    "  # 6. initialize the text classifier trainer\n",
    "  trainer = ModelTrainer(classifier,\n",
    "                          corpus,\n",
    "                          optimizer=Adam\n",
    "                          )\n",
    "\n",
    "\n",
    "  trainer.train(base_path,\n",
    "                learning_rate=0.03,\n",
    "                mini_batch_size=16,\n",
    "                anneal_factor=0.5,\n",
    "                patience=5,\n",
    "                max_epochs=10,\n",
    "                checkpoint=True,\n",
    "                embeddings_storage_mode='gpu'\n",
    "                )"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "5_model_training.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}