TF2_Intent_Classification_DistilBERT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.3"
    },
    "colab": {
      "name": "TF2_Intent_Classification_DistilBERT.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "6888166f306d4a17a2ba749e3b4f4a0b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_393301ffd2cd480ab810dfda58e85064",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9fd86021b854427c83c117f728296b4b",
              "IPY_MODEL_8e9c9531c1e3499083c20bafe2163ba4"
            ]
          }
        },
        "393301ffd2cd480ab810dfda58e85064": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9fd86021b854427c83c117f728296b4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e0feeb00e14e4f9ab3089743475c5705",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1b3cbea8bae140d4af0dad4a945d5693"
          }
        },
        "8e9c9531c1e3499083c20bafe2163ba4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b27f8c41895547a4a34ac44c3f2d048d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:01&lt;00:00, 181kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f4ef6d096b194a69a4615be5928ad708"
          }
        },
        "e0feeb00e14e4f9ab3089743475c5705": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1b3cbea8bae140d4af0dad4a945d5693": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b27f8c41895547a4a34ac44c3f2d048d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f4ef6d096b194a69a4615be5928ad708": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "feb3e0c1eba14bf594625dd834567794": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_71089e10684e47698204ba8849d49550",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_61224aad6e63451e858bb4df77a2bcbc",
              "IPY_MODEL_c603af1cdbc146b4a0427000c6baa78b"
            ]
          }
        },
        "71089e10684e47698204ba8849d49550": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "61224aad6e63451e858bb4df77a2bcbc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c22869965c544985bcd5f049c51f993d",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_32500c1046c4476686917ccc2252634c"
          }
        },
        "c603af1cdbc146b4a0427000c6baa78b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_edab717f59b947bcb44b88ab43b0d9ac",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:00&lt;00:00, 33.0B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_50282e6b74684063b12a2b4827bcc35f"
          }
        },
        "c22869965c544985bcd5f049c51f993d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "32500c1046c4476686917ccc2252634c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "edab717f59b947bcb44b88ab43b0d9ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "50282e6b74684063b12a2b4827bcc35f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8c393f12b3504cbfa03af1d5eeb6c7eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f505e26352094006975bb4bb6e96469e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_edb3049d16ce4648918d95ec011bd82c",
              "IPY_MODEL_9a9f1e5a98c44afa9bf7e0f680f7c84a"
            ]
          }
        },
        "f505e26352094006975bb4bb6e96469e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "edb3049d16ce4648918d95ec011bd82c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5e62319a4c4846f895ef740c96a4c6a5",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_abca75c50f554522a8953998361d256d"
          }
        },
        "9a9f1e5a98c44afa9bf7e0f680f7c84a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_226f0b933153470392d762b6cbce6126",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:00&lt;00:00, 2.16MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_659b1c1dc2e34fba8ea520a337049ae0"
          }
        },
        "5e62319a4c4846f895ef740c96a4c6a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "abca75c50f554522a8953998361d256d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "226f0b933153470392d762b6cbce6126": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "659b1c1dc2e34fba8ea520a337049ae0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rajesh-bhat/data-aisummit-2021-databricks-conversational-ai/blob/main/TF2_Intent_Classification_DistilBERT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hp78pgLek3CG",
        "outputId": "bd141bb0-e9a8-4312-e07e-63972eb0e3c6"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Thu May 27 17:21:04 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 465.19.01    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   54C    P8    10W /  70W |      0MiB / 15109MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uP1MojEU1RKV",
        "outputId": "0e6c0b85-8790-4dc3-9f96-eee63910cd65"
      },
      "source": [
        "!pip3 install tqdm\n",
        "!pip3 install transformers"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (4.41.1)\n",
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d5/43/cfe4ee779bbd6a678ac6a97c5a5cdeb03c35f9eaebbb9720b036680f9a2d/transformers-4.6.1-py3-none-any.whl (2.2MB)\n",
            "\u001b[K     |████████████████████████████████| 2.3MB 7.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (4.0.1)\n",
            "Collecting huggingface-hub==0.0.8\n",
            "  Downloading https://files.pythonhosted.org/packages/a1/88/7b1e45720ecf59c6c6737ff332f41c955963090a18e72acbcbeac6b25e86/huggingface_hub-0.0.8-py3-none-any.whl\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 43.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/e2/df3543e8ffdab68f5acc73f613de9c2b155ac47f162e725dcac87c521c11/tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 48.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Installing collected packages: huggingface-hub, sacremoses, tokenizers, transformers\n",
            "Successfully installed huggingface-hub-0.0.8 sacremoses-0.0.45 tokenizers-0.10.3 transformers-4.6.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "be5R1UOO1K-f"
      },
      "source": [
        "## Downloading the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7z1TW64FT--O",
        "outputId": "443de396-9f85-4ca8-c6fe-ed750392b3ae"
      },
      "source": [
        "!gdown --id 1OlcvGWReJMuyYQuOZm149vHWwPtlboR6 --output train.csv\n",
        "!gdown --id 1Oi5cRlTybuIF2Fl5Bfsr-KkqrXrdt77w --output valid.csv\n",
        "!gdown --id 1ep9H6-HvhB4utJRLVcLzieWNUSG3P_uF --output test.csv"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1OlcvGWReJMuyYQuOZm149vHWwPtlboR6\n",
            "To: /content/train.csv\n",
            "100% 799k/799k [00:00<00:00, 52.4MB/s]\n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1Oi5cRlTybuIF2Fl5Bfsr-KkqrXrdt77w\n",
            "To: /content/valid.csv\n",
            "100% 43.3k/43.3k [00:00<00:00, 67.1MB/s]\n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1ep9H6-HvhB4utJRLVcLzieWNUSG3P_uF\n",
            "To: /content/test.csv\n",
            "100% 43.1k/43.1k [00:00<00:00, 41.6MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1wWodPdPk3CJ",
        "outputId": "66f4b662-3a27-435c-8fe1-f31953014ec1"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import transformers\n",
        "import matplotlib.pyplot as plt\n",
        "import tqdm\n",
        "from sklearn.preprocessing import LabelBinarizer\n",
        "%matplotlib inline\n",
        "\n",
        "# fix random seed for reproducibility\n",
        "seed = 222\n",
        "np.random.seed(seed)\n",
        "tf.random.set_seed(seed)\n",
        "\n",
        "print(\"TF Version: \", tf.__version__)\n",
        "print(\"Eager mode: \", tf.executing_eagerly())\n",
        "print(\"GPU is\", \"available\" if tf.test.is_gpu_available() else \"NOT AVAILABLE\")"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TF Version:  2.5.0\n",
            "Eager mode:  True\n",
            "WARNING:tensorflow:From <ipython-input-4-9b8022030b98>:17: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.config.list_physical_devices('GPU')` instead.\n",
            "GPU is available\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6f-yedJWUK4t"
      },
      "source": [
        "##Overview of data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8Vt68BI_UKTy",
        "outputId": "ba354618-f0aa-47e4-afd6-c338621872c7"
      },
      "source": [
        "train = pd.concat([pd.read_csv(file) for file in [\"train.csv\"]])\n",
        "train = train.groupby('intent').sample(frac=0.25)\n",
        "\n",
        "val = pd.read_csv(\"valid.csv\")\n",
        "\n",
        "print(train.shape)\n",
        "print(val.shape)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(3270, 2)\n",
            "(700, 2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "sj98b3n-2Vmc",
        "outputId": "07adb028-c3dd-4c0e-8281-b9b29bd62fa3"
      },
      "source": [
        "train.head()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>intent</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4024</th>\n",
              "      <td>can you put this tune onto latin dance cardio</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4592</th>\n",
              "      <td>marcia carolus rex should get added to my wome...</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9153</th>\n",
              "      <td>add r u still in 2 it to the under the surface...</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11241</th>\n",
              "      <td>add a billy strayhorn song to my the selektor ...</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8989</th>\n",
              "      <td>add an album to women of hip hop</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text         intent\n",
              "4024       can you put this tune onto latin dance cardio  AddToPlaylist\n",
              "4592   marcia carolus rex should get added to my wome...  AddToPlaylist\n",
              "9153   add r u still in 2 it to the under the surface...  AddToPlaylist\n",
              "11241  add a billy strayhorn song to my the selektor ...  AddToPlaylist\n",
              "8989                    add an album to women of hip hop  AddToPlaylist"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "txVCTzUDUw7n",
        "outputId": "ca79d6b3-7e48-4d2e-80c7-b9fea2360982"
      },
      "source": [
        "val.head()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>intent</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>i d like to have this track onto my classical ...</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>add the album to my flow español playlist</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>add digging now to my young at heart playlist</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>add this song by too poetic to my piano ballad...</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>add this album to old school death metal</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text         intent\n",
              "0  i d like to have this track onto my classical ...  AddToPlaylist\n",
              "1          add the album to my flow español playlist  AddToPlaylist\n",
              "2      add digging now to my young at heart playlist  AddToPlaylist\n",
              "3  add this song by too poetic to my piano ballad...  AddToPlaylist\n",
              "4           add this album to old school death metal  AddToPlaylist"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sK5YLZTVUcql",
        "outputId": "d85a1412-9843-452e-c715-5b3779806579"
      },
      "source": [
        "train.intent.value_counts()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "PlayMusic               478\n",
              "GetWeather              474\n",
              "BookRestaurant          470\n",
              "RateBook                469\n",
              "SearchScreeningEvent    463\n",
              "SearchCreativeWork      462\n",
              "AddToPlaylist           454\n",
              "Name: intent, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "elBnbqiO2fmL"
      },
      "source": [
        "## Label Encoded target variables"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "J7HS0k_kUf4v",
        "outputId": "c7248221-dce2-478a-f575-31559bbe657d"
      },
      "source": [
        "intent_mapping = {x:idx for idx,x in enumerate(train.intent.unique().tolist())}\n",
        "intent_mapping_rev = {v: k for k, v in intent_mapping.items()}\n",
        "\n",
        "train['target'] = train['intent'].map(intent_mapping)\n",
        "val['target'] = val['intent'].map(intent_mapping)\n",
        "\n",
        "train.head()"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>intent</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4024</th>\n",
              "      <td>can you put this tune onto latin dance cardio</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4592</th>\n",
              "      <td>marcia carolus rex should get added to my wome...</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9153</th>\n",
              "      <td>add r u still in 2 it to the under the surface...</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11241</th>\n",
              "      <td>add a billy strayhorn song to my the selektor ...</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8989</th>\n",
              "      <td>add an album to women of hip hop</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text         intent  target\n",
              "4024       can you put this tune onto latin dance cardio  AddToPlaylist       0\n",
              "4592   marcia carolus rex should get added to my wome...  AddToPlaylist       0\n",
              "9153   add r u still in 2 it to the under the surface...  AddToPlaylist       0\n",
              "11241  add a billy strayhorn song to my the selektor ...  AddToPlaylist       0\n",
              "8989                    add an album to women of hip hop  AddToPlaylist       0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EJlhBGzS8-j-",
        "outputId": "0aa0b075-30bd-42a2-902a-e887611a079e"
      },
      "source": [
        "intent_mapping, intent_mapping_rev"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'AddToPlaylist': 0,\n",
              "  'BookRestaurant': 1,\n",
              "  'GetWeather': 2,\n",
              "  'PlayMusic': 3,\n",
              "  'RateBook': 4,\n",
              "  'SearchCreativeWork': 5,\n",
              "  'SearchScreeningEvent': 6},\n",
              " {0: 'AddToPlaylist',\n",
              "  1: 'BookRestaurant',\n",
              "  2: 'GetWeather',\n",
              "  3: 'PlayMusic',\n",
              "  4: 'RateBook',\n",
              "  5: 'SearchCreativeWork',\n",
              "  6: 'SearchScreeningEvent'})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "xOZIPZkKVBqt",
        "outputId": "fa37ed98-e11f-4fd7-c88e-3ce01fc9cf41"
      },
      "source": [
        "val.head()    "
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>intent</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>i d like to have this track onto my classical ...</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>add the album to my flow español playlist</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>add digging now to my young at heart playlist</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>add this song by too poetic to my piano ballad...</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>add this album to old school death metal</td>\n",
              "      <td>AddToPlaylist</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text         intent  target\n",
              "0  i d like to have this track onto my classical ...  AddToPlaylist       0\n",
              "1          add the album to my flow español playlist  AddToPlaylist       0\n",
              "2      add digging now to my young at heart playlist  AddToPlaylist       0\n",
              "3  add this song by too poetic to my piano ballad...  AddToPlaylist       0\n",
              "4           add this album to old school death metal  AddToPlaylist       0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u0hKB1UV4KyD"
      },
      "source": [
        "## One hot encoding the labels"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ORgVqUO94JiG",
        "outputId": "b669f9ee-ece4-4931-d69d-e1d9439785b8"
      },
      "source": [
        "lb = LabelBinarizer( )\n",
        "lb.fit(train[\"target\"])\n",
        "print(lb.classes_)\n",
        "\n",
        "y_train = lb.transform(train[\"target\"])\n",
        "y_val = lb.transform(val[\"target\"])"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[0 1 2 3 4 5 6]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jpAc91NWk3CL"
      },
      "source": [
        "def create_bert_input_features(tokenizer, docs, max_seq_length):\n",
        "    '''\n",
        "    tokenizer: based on the model chosen\n",
        "    docs: sentences/utterances\n",
        "    max_seq_length: used for padding\n",
        "    '''\n",
        "\n",
        "    all_ids, all_masks = [], []\n",
        "\n",
        "    for doc in tqdm.tqdm(docs, desc=\"Converting sentences to features\"):\n",
        "        \n",
        "        tokens = tokenizer.tokenize(doc)\n",
        "\n",
        "        if len(tokens) > max_seq_length-2:\n",
        "            tokens = tokens[0 : (max_seq_length-2)]\n",
        "        \n",
        "        tokens = ['[CLS]'] + tokens + ['[SEP]']\n",
        "        ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "        masks = [1] * len(ids)\n",
        "\n",
        "        # zero-padding up to the sequence length.\n",
        "        while len(ids) < max_seq_length:\n",
        "            ids.append(0)\n",
        "            masks.append(0)\n",
        "        \n",
        "        all_ids.append(ids)\n",
        "        all_masks.append(masks)\n",
        "    \n",
        "    ## encoded sentence along with masks.\n",
        "    encoded = np.array([all_ids, all_masks])\n",
        "\n",
        "    return encoded"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rBaCLzra3QEH"
      },
      "source": [
        "## Load DistilBertTokenizer from transformers module"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164,
          "referenced_widgets": [
            "6888166f306d4a17a2ba749e3b4f4a0b",
            "393301ffd2cd480ab810dfda58e85064",
            "9fd86021b854427c83c117f728296b4b",
            "8e9c9531c1e3499083c20bafe2163ba4",
            "e0feeb00e14e4f9ab3089743475c5705",
            "1b3cbea8bae140d4af0dad4a945d5693",
            "b27f8c41895547a4a34ac44c3f2d048d",
            "f4ef6d096b194a69a4615be5928ad708",
            "feb3e0c1eba14bf594625dd834567794",
            "71089e10684e47698204ba8849d49550",
            "61224aad6e63451e858bb4df77a2bcbc",
            "c603af1cdbc146b4a0427000c6baa78b",
            "c22869965c544985bcd5f049c51f993d",
            "32500c1046c4476686917ccc2252634c",
            "edab717f59b947bcb44b88ab43b0d9ac",
            "50282e6b74684063b12a2b4827bcc35f",
            "8c393f12b3504cbfa03af1d5eeb6c7eb",
            "f505e26352094006975bb4bb6e96469e",
            "edb3049d16ce4648918d95ec011bd82c",
            "9a9f1e5a98c44afa9bf7e0f680f7c84a",
            "5e62319a4c4846f895ef740c96a4c6a5",
            "abca75c50f554522a8953998361d256d",
            "226f0b933153470392d762b6cbce6126",
            "659b1c1dc2e34fba8ea520a337049ae0"
          ]
        },
        "id": "tFCHA-mWk3CK",
        "outputId": "b2e94e98-a909-4fd8-d5c9-51f946f41352"
      },
      "source": [
        "tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-uncased')"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6888166f306d4a17a2ba749e3b4f4a0b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "feb3e0c1eba14bf594625dd834567794",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8c393f12b3504cbfa03af1d5eeb6c7eb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xpvqkur635jC"
      },
      "source": [
        "## Model parameters and summary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cAEe7gsHk3CL",
        "outputId": "7d1c3b5d-6953-4c08-f21b-7c11ad4eee8d"
      },
      "source": [
        "MAX_SEQ_LENGTH = 256 ## can be chosen based on the sentence length distribution\n",
        "num_labels = 7 ## with the current data we have 7 classes\n",
        "\n",
        "inp_id = tf.keras.layers.Input(shape=(MAX_SEQ_LENGTH,), dtype='int32', name=\"bert_input_ids\")\n",
        "inp_mask = tf.keras.layers.Input(shape=(MAX_SEQ_LENGTH,), dtype='int32', name=\"bert_input_masks\")\n",
        "inputs = [inp_id, inp_mask]\n",
        "\n",
        "hidden_state = transformers.TFDistilBertModel.from_pretrained('distilbert-base-uncased')(inputs)[0]\n",
        "pooled_output = hidden_state[:, 0]\n",
        "\n",
        "## dense layers\n",
        "dense1 = tf.keras.layers.Dense(256, activation='relu')(pooled_output)\n",
        "drop1 = tf.keras.layers.Dropout(0.25)(dense1)\n",
        "dense2 = tf.keras.layers.Dense(256, activation='relu')(drop1)\n",
        "drop2 = tf.keras.layers.Dropout(0.25)(dense2)\n",
        "output = tf.keras.layers.Dense(num_labels, activation='softmax')(drop2)\n",
        "\n",
        "model = tf.keras.Model(inputs=inputs, outputs=output)\n",
        "model.compile(optimizer=tf.optimizers.Adam(learning_rate=2e-5, \n",
        "                                           epsilon=1e-08), \n",
        "              loss='categorical_crossentropy', \n",
        "              metrics=['accuracy'])\n",
        "\n",
        "model.summary()"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertModel: ['activation_13', 'vocab_projector', 'vocab_transform', 'vocab_layer_norm']\n",
            "- This IS expected if you are initializing TFDistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFDistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFDistilBertModel were initialized from the model checkpoint at distilbert-base-uncased.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFDistilBertModel for predictions without further training.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "bert_input_ids (InputLayer)     [(None, 256)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "bert_input_masks (InputLayer)   [(None, 256)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "tf_distil_bert_model_1 (TFDisti TFBaseModelOutput(la 66362880    bert_input_ids[0][0]             \n",
            "                                                                 bert_input_masks[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "tf.__operators__.getitem_1 (Sli (None, 768)          0           tf_distil_bert_model_1[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 256)          196864      tf.__operators__.getitem_1[0][0] \n",
            "__________________________________________________________________________________________________\n",
            "dropout_40 (Dropout)            (None, 256)          0           dense_3[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_4 (Dense)                 (None, 256)          65792       dropout_40[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "dropout_41 (Dropout)            (None, 256)          0           dense_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_5 (Dense)                 (None, 7)            1799        dropout_41[0][0]                 \n",
            "==================================================================================================\n",
            "Total params: 66,627,335\n",
            "Trainable params: 66,627,335\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0StkQRufk3CL",
        "outputId": "ebf56425-c5e9-4c38-c3b4-713d168d5d4b"
      },
      "source": [
        "train_features_ids, train_features_masks = create_bert_input_features(tokenizer, train[\"text\"], max_seq_length=MAX_SEQ_LENGTH)\n",
        "val_features_ids, val_features_masks = create_bert_input_features(tokenizer, val[\"text\"], max_seq_length=MAX_SEQ_LENGTH)\n",
        "\n",
        "print('Train Features:', train_features_ids.shape, train_features_masks.shape)\n",
        "print('Val Features:', val_features_ids.shape, val_features_masks.shape)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Converting docs to features: 100%|██████████| 3270/3270 [00:01<00:00, 2874.38it/s]\n",
            "Converting docs to features: 100%|██████████| 700/700 [00:00<00:00, 3393.64it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train Features: (3270, 256) (3270, 256)\n",
            "Val Features: (700, 256) (700, 256)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oQvp7uxhaY9C",
        "outputId": "2840f1a0-826e-4547-a6ee-9c4b92a1ebfb"
      },
      "source": [
        "print(\"Target shape\", y_train.shape, y_val.shape)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Target shape (3270, 7) (700, 7)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "20N4uYtnk3CM",
        "outputId": "fc38b703-cc42-4017-b460-56b3508d347f"
      },
      "source": [
        "es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', \n",
        "                                      patience=3,\n",
        "                                      restore_best_weights=True,\n",
        "                                      verbose=1)\n",
        "\n",
        "checkpoint = tf.keras.callbacks.ModelCheckpoint('tf_intent_classification.hdf5', save_best_only=True, monitor='val_accuracy', mode='max')\n",
        "\n",
        "model.fit([train_features_ids, \n",
        "           train_features_masks], y_train, \n",
        "          validation_data=([val_features_ids, \n",
        "                            val_features_masks], y_val),\n",
        "          epochs=3, \n",
        "          batch_size=32, \n",
        "          shuffle=True,\n",
        "          callbacks=[es],\n",
        "          verbose=1)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "103/103 [==============================] - 86s 835ms/step - loss: 0.1064 - accuracy: 0.9719 - val_loss: 0.0738 - val_accuracy: 0.9771\n",
            "Epoch 2/3\n",
            "103/103 [==============================] - 86s 834ms/step - loss: 0.0538 - accuracy: 0.9902 - val_loss: 0.1089 - val_accuracy: 0.9700\n",
            "Epoch 3/3\n",
            "103/103 [==============================] - 86s 831ms/step - loss: 0.0354 - accuracy: 0.9930 - val_loss: 0.0791 - val_accuracy: 0.9800\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f9eb824e5d0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Um7ZkH68k3CN"
      },
      "source": [
        "def score(sentence):\n",
        "    '''\n",
        "    input:sentence\n",
        "    return: intent\n",
        "    '''\n",
        "\n",
        "    features_ids, features_masks = create_bert_input_features(tokenizer, [sentence], max_seq_length=MAX_SEQ_LENGTH)\n",
        "    predictions = model.predict([features_ids, features_masks], batch_size=1, verbose=0)\n",
        "    print(predictions)\n",
        "    print(np.argmax(predictions))\n",
        "\n",
        "    return {\"query\":sentence, \"intent\":intent_mapping_rev[np.argmax(predictions)]}"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6ehvKDlg8e0i",
        "outputId": "19f6486e-6f6b-4999-b3d2-85735bb3ccb8"
      },
      "source": [
        "score(\"book a restaurant near my office\")"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Converting sentences to features: 100%|██████████| 1/1 [00:00<00:00, 943.60it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[[1.4433196e-04 9.9869776e-01 4.9405068e-04 3.5746428e-04 1.8700135e-04\n",
            "  5.1930572e-05 6.7458204e-05]]\n",
            "1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'intent': 'BookRestaurant', 'query': 'book a restaurant near my office'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6gWCAuCV78sD"
      },
      "source": [
        "References:\n",
        "<br>\n",
        "[1] Sarkar, Dipanjan (DJ). DipanjanS/Deep_transfer_learning_nlp_dhs2019. 2019. 2021. GitHub, https://github.com/dipanjanS/deep_transfer_learning_nlp_dhs2019"
      ]
    }
  ]
}