BERT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b9eb4eed50174e4fb5e8cdd8e342f8a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2adb9349be15429097ae3c6983283975",
              "IPY_MODEL_3115c681901742479247ac8cc319f6ab",
              "IPY_MODEL_60e6cfce53ef4410aa0c20bbd135ba98"
            ],
            "layout": "IPY_MODEL_c8567b3582ad435b9af1ffa24147fd22"
          }
        },
        "2adb9349be15429097ae3c6983283975": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0473282cc9a140a0bb28aa252aaff0a0",
            "placeholder": "​",
            "style": "IPY_MODEL_f0d562b47bd6438196e15394e0a28b67",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "3115c681901742479247ac8cc319f6ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_73b4a2d35e4648aeab698ea47b650a48",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_922665774c16451cba89e0900fcee46f",
            "value": 570
          }
        },
        "60e6cfce53ef4410aa0c20bbd135ba98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9aa27d05552f493682ee651654c2c4d3",
            "placeholder": "​",
            "style": "IPY_MODEL_a09d67e066c3426c99d3882d45e8461d",
            "value": " 570/570 [00:00&lt;00:00, 20.4kB/s]"
          }
        },
        "c8567b3582ad435b9af1ffa24147fd22": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0473282cc9a140a0bb28aa252aaff0a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f0d562b47bd6438196e15394e0a28b67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "73b4a2d35e4648aeab698ea47b650a48": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "922665774c16451cba89e0900fcee46f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9aa27d05552f493682ee651654c2c4d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a09d67e066c3426c99d3882d45e8461d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3933ce313e0e4dcfb3e3fe504b000cdd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9067620e3045437fbc29fc9dc329bad8",
              "IPY_MODEL_a79d95cb6ebd4253b05575a0d3c58f19",
              "IPY_MODEL_7588b9d98a8f4b858f88015eb7aa12fb"
            ],
            "layout": "IPY_MODEL_c013bf2759384b0db966eaf7d717ef9f"
          }
        },
        "9067620e3045437fbc29fc9dc329bad8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_81bff108ae3b42c7b508b6bff825bcdd",
            "placeholder": "​",
            "style": "IPY_MODEL_05bb691b372e4f48a6547446e1ece474",
            "value": "Downloading model.safetensors: 100%"
          }
        },
        "a79d95cb6ebd4253b05575a0d3c58f19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ce4ec863ef73435cb00eae48180722c7",
            "max": 440449768,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e7c6c2b4c6f2491385a504d1001e5cb8",
            "value": 440449768
          }
        },
        "7588b9d98a8f4b858f88015eb7aa12fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_384ae7ad4827413593545c74ef9b7cb0",
            "placeholder": "​",
            "style": "IPY_MODEL_6b9fc4a964624468a5c6c5e1429b15c2",
            "value": " 440M/440M [00:05&lt;00:00, 99.2MB/s]"
          }
        },
        "c013bf2759384b0db966eaf7d717ef9f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "81bff108ae3b42c7b508b6bff825bcdd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "05bb691b372e4f48a6547446e1ece474": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ce4ec863ef73435cb00eae48180722c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e7c6c2b4c6f2491385a504d1001e5cb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "384ae7ad4827413593545c74ef9b7cb0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6b9fc4a964624468a5c6c5e1429b15c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "63b8f71907f04f5499bbe72594495cb3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8b9cbc9f026b450286357e290ccf3fb8",
              "IPY_MODEL_6ce6b15bd00543839877150a17503758",
              "IPY_MODEL_a76c79f5d4724c89bf2da445a5a8699a"
            ],
            "layout": "IPY_MODEL_17ef4af4a4ba4d80930e470afb314424"
          }
        },
        "8b9cbc9f026b450286357e290ccf3fb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_56c1292f202c4ce2a22685cab6f8cd34",
            "placeholder": "​",
            "style": "IPY_MODEL_ad0bd474529249a7a9a1dc9d487c1967",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "6ce6b15bd00543839877150a17503758": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_14a9a8a9e2cc4064b3ba49c68524bd42",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_941e60a745c043bcbd3e1ef8e5979d29",
            "value": 28
          }
        },
        "a76c79f5d4724c89bf2da445a5a8699a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c9178f51e44a4d00a1f8241b0d565c40",
            "placeholder": "​",
            "style": "IPY_MODEL_09599fc3e6074e24acc0732ad4150909",
            "value": " 28.0/28.0 [00:00&lt;00:00, 1.05kB/s]"
          }
        },
        "17ef4af4a4ba4d80930e470afb314424": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "56c1292f202c4ce2a22685cab6f8cd34": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ad0bd474529249a7a9a1dc9d487c1967": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "14a9a8a9e2cc4064b3ba49c68524bd42": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "941e60a745c043bcbd3e1ef8e5979d29": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c9178f51e44a4d00a1f8241b0d565c40": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "09599fc3e6074e24acc0732ad4150909": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "673f49a7622f41ebb236f8199214ca7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8d66c4ccd38348d4818ee161d3bc36ba",
              "IPY_MODEL_4f9bcd7f8a38449f87f918beab95bea6",
              "IPY_MODEL_aeece01d7cef4cbf8c5f86bfc94bb00e"
            ],
            "layout": "IPY_MODEL_f1c2476ad408400e8b1014c9a7285fb2"
          }
        },
        "8d66c4ccd38348d4818ee161d3bc36ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5efa4a88decb49419ed1047e37726046",
            "placeholder": "​",
            "style": "IPY_MODEL_2dc0aee3fa844de38623e48856e76c16",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "4f9bcd7f8a38449f87f918beab95bea6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4554f8e64db54c4c8408af216bfb7319",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_63d879b9421b48d38ac225b12b4d60a6",
            "value": 231508
          }
        },
        "aeece01d7cef4cbf8c5f86bfc94bb00e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_53ef815d80914418aadd0158040b2835",
            "placeholder": "​",
            "style": "IPY_MODEL_d7e2ad77963a47d6a8442109d458fdcf",
            "value": " 232k/232k [00:00&lt;00:00, 1.73MB/s]"
          }
        },
        "f1c2476ad408400e8b1014c9a7285fb2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5efa4a88decb49419ed1047e37726046": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2dc0aee3fa844de38623e48856e76c16": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4554f8e64db54c4c8408af216bfb7319": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "63d879b9421b48d38ac225b12b4d60a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "53ef815d80914418aadd0158040b2835": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d7e2ad77963a47d6a8442109d458fdcf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4be0f98adbc546778cc0e5b1cc660566": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9a403deb50d4439db92a109720c13ead",
              "IPY_MODEL_502def7f14ed4eb9811affbc4a9e2dfd",
              "IPY_MODEL_e287f2a6a4c749fca0fd23ccc6a62b59"
            ],
            "layout": "IPY_MODEL_0a04a83d14ee488fa13319ae7f3f6c72"
          }
        },
        "9a403deb50d4439db92a109720c13ead": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7319ebc3c5c44cb9b0c892495faf480a",
            "placeholder": "​",
            "style": "IPY_MODEL_d511198999a7446da1834022c8408f94",
            "value": "Downloading (…)/main/tokenizer.json: 100%"
          }
        },
        "502def7f14ed4eb9811affbc4a9e2dfd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dd59efa48997494ab82b15d324843811",
            "max": 466062,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_71826ce129b549a4b7364246a2230b3f",
            "value": 466062
          }
        },
        "e287f2a6a4c749fca0fd23ccc6a62b59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2a8ac884ba2148218c7bc06c1541337d",
            "placeholder": "​",
            "style": "IPY_MODEL_a98db6dc9d5e442e9f99054f3e875bff",
            "value": " 466k/466k [00:00&lt;00:00, 2.35MB/s]"
          }
        },
        "0a04a83d14ee488fa13319ae7f3f6c72": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7319ebc3c5c44cb9b0c892495faf480a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d511198999a7446da1834022c8408f94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dd59efa48997494ab82b15d324843811": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71826ce129b549a4b7364246a2230b3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2a8ac884ba2148218c7bc06c1541337d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a98db6dc9d5e442e9f99054f3e875bff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "**Problem Statement**\n",
        "\n",
        "The main objective is to create a system with the BERT model for classifying SMS messages as spam or non-spam to enhance potential security."
      ],
      "metadata": {
        "id": "G1wzdyo229cM"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Import required libraries and dataset**\n"
      ],
      "metadata": {
        "id": "qPb3q45kbfsJ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CuG2KBbxSu8f",
        "outputId": "f73b237a-c945-4aff-c81e-b0eec784444e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers\n",
            "  Using cached transformers-4.34.1-py3-none-any.whl (7.7 MB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.4)\n",
            "Collecting huggingface-hub<1.0,>=0.16.4 (from transformers)\n",
            "  Using cached huggingface_hub-0.18.0-py3-none-any.whl (301 kB)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Collecting tokenizers<0.15,>=0.14 (from transformers)\n",
            "  Using cached tokenizers-0.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n",
            "Collecting safetensors>=0.3.1 (from transformers)\n",
            "  Using cached safetensors-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (4.5.0)\n",
            "Collecting huggingface-hub<1.0,>=0.16.4 (from transformers)\n",
            "  Using cached huggingface_hub-0.17.3-py3-none-any.whl (295 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.7.22)\n",
            "Installing collected packages: safetensors, huggingface-hub, tokenizers, transformers\n",
            "Successfully installed huggingface-hub-0.17.3 safetensors-0.4.0 tokenizers-0.14.1 transformers-4.34.1\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import classification_report\n",
        "import transformers\n",
        "from transformers import AutoModel, BertTokenizerFast\n",
        "\n",
        "# specify GPU\n",
        "#device = torch.device(\"cuda\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "\n",
        "uploaded = files.upload()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        },
        "id": "gHq_vcC3Vpgt",
        "outputId": "6bfcd3ce-d24e-42b1-cad2-8f34c234463c"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-4af57be2-0f3a-44f1-b5c5-5d1c42b7149c\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-4af57be2-0f3a-44f1-b5c5-5d1c42b7149c\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving spam.csv to spam.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(\"spam.csv\", encoding = 'latin-1')\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "p4SkDxM_WIzL",
        "outputId": "94568d8e-52a9-42fb-c32e-dcf684870375"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     v1                                                 v2 Unnamed: 2  \\\n",
              "0   ham  Go until jurong point, crazy.. Available only ...        NaN   \n",
              "1   ham                      Ok lar... Joking wif u oni...        NaN   \n",
              "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...        NaN   \n",
              "3   ham  U dun say so early hor... U c already then say...        NaN   \n",
              "4   ham  Nah I don't think he goes to usf, he lives aro...        NaN   \n",
              "\n",
              "  Unnamed: 3 Unnamed: 4  \n",
              "0        NaN        NaN  \n",
              "1        NaN        NaN  \n",
              "2        NaN        NaN  \n",
              "3        NaN        NaN  \n",
              "4        NaN        NaN  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e7f189a0-5b3f-4f68-8204-d9f51d8aa2e9\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>v1</th>\n",
              "      <th>v2</th>\n",
              "      <th>Unnamed: 2</th>\n",
              "      <th>Unnamed: 3</th>\n",
              "      <th>Unnamed: 4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ham</td>\n",
              "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ham</td>\n",
              "      <td>Ok lar... Joking wif u oni...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>spam</td>\n",
              "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ham</td>\n",
              "      <td>U dun say so early hor... U c already then say...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ham</td>\n",
              "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e7f189a0-5b3f-4f68-8204-d9f51d8aa2e9')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-e7f189a0-5b3f-4f68-8204-d9f51d8aa2e9 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-e7f189a0-5b3f-4f68-8204-d9f51d8aa2e9');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-563f1c55-f30f-4fbb-ad2f-5e82bdd8a166\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-563f1c55-f30f-4fbb-ad2f-5e82bdd8a166')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-563f1c55-f30f-4fbb-ad2f-5e82bdd8a166 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.dropna(how=\"any\", inplace=True, axis=1)\n",
        "df.columns = ['label', 'message']\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "zDO_uwANYiZB",
        "outputId": "9e01c4f0-436b-4dc5-c5ae-09e4b2db40ef"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  label                                            message\n",
              "0   ham  Go until jurong point, crazy.. Available only ...\n",
              "1   ham                      Ok lar... Joking wif u oni...\n",
              "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
              "3   ham  U dun say so early hor... U c already then say...\n",
              "4   ham  Nah I don't think he goes to usf, he lives aro..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5fc835d2-844b-4bba-9610-60ba62bbd6a1\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>message</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ham</td>\n",
              "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ham</td>\n",
              "      <td>Ok lar... Joking wif u oni...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>spam</td>\n",
              "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ham</td>\n",
              "      <td>U dun say so early hor... U c already then say...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ham</td>\n",
              "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5fc835d2-844b-4bba-9610-60ba62bbd6a1')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-5fc835d2-844b-4bba-9610-60ba62bbd6a1 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-5fc835d2-844b-4bba-9610-60ba62bbd6a1');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-e47e7f81-61c3-4040-97be-46d6455defd7\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-e47e7f81-61c3-4040-97be-46d6455defd7')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-e47e7f81-61c3-4040-97be-46d6455defd7 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['label_num']=df.label.map({'ham':0,'spam':1})\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "DOU9e_fdY6IH",
        "outputId": "e3202096-d3ae-4ca0-afda-9357611a9927"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  label                                            message  label_num\n",
              "0   ham  Go until jurong point, crazy.. Available only ...          0\n",
              "1   ham                      Ok lar... Joking wif u oni...          0\n",
              "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...          1\n",
              "3   ham  U dun say so early hor... U c already then say...          0\n",
              "4   ham  Nah I don't think he goes to usf, he lives aro...          0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d4b4c66e-5087-4c54-9662-3463f1b19a13\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>message</th>\n",
              "      <th>label_num</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ham</td>\n",
              "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ham</td>\n",
              "      <td>Ok lar... Joking wif u oni...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>spam</td>\n",
              "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ham</td>\n",
              "      <td>U dun say so early hor... U c already then say...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ham</td>\n",
              "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d4b4c66e-5087-4c54-9662-3463f1b19a13')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-d4b4c66e-5087-4c54-9662-3463f1b19a13 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-d4b4c66e-5087-4c54-9662-3463f1b19a13');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-7120a490-a85c-412b-afc3-53b3a31880fb\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7120a490-a85c-412b-afc3-53b3a31880fb')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-7120a490-a85c-412b-afc3-53b3a31880fb button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['msglength']=df.message.apply(len)\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "lH4xOUhPZHx3",
        "outputId": "a7c1a463-d920-48a7-e64a-f5a1c6c2990e"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  label                                            message  label_num  \\\n",
              "0   ham  Go until jurong point, crazy.. Available only ...          0   \n",
              "1   ham                      Ok lar... Joking wif u oni...          0   \n",
              "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...          1   \n",
              "3   ham  U dun say so early hor... U c already then say...          0   \n",
              "4   ham  Nah I don't think he goes to usf, he lives aro...          0   \n",
              "\n",
              "   msglength  \n",
              "0        111  \n",
              "1         29  \n",
              "2        155  \n",
              "3         49  \n",
              "4         61  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-bad4f917-7e6b-40bb-87ed-1edf52954aff\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>message</th>\n",
              "      <th>label_num</th>\n",
              "      <th>msglength</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ham</td>\n",
              "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
              "      <td>0</td>\n",
              "      <td>111</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ham</td>\n",
              "      <td>Ok lar... Joking wif u oni...</td>\n",
              "      <td>0</td>\n",
              "      <td>29</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>spam</td>\n",
              "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
              "      <td>1</td>\n",
              "      <td>155</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ham</td>\n",
              "      <td>U dun say so early hor... U c already then say...</td>\n",
              "      <td>0</td>\n",
              "      <td>49</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ham</td>\n",
              "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
              "      <td>0</td>\n",
              "      <td>61</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bad4f917-7e6b-40bb-87ed-1edf52954aff')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-bad4f917-7e6b-40bb-87ed-1edf52954aff button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-bad4f917-7e6b-40bb-87ed-1edf52954aff');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-bb7019e9-13a5-4ba2-84b3-477cacf0d7b4\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-bb7019e9-13a5-4ba2-84b3-477cacf0d7b4')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-bb7019e9-13a5-4ba2-84b3-477cacf0d7b4 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# check class distribution\n",
        "df['label_num'].value_counts(normalize = True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D2LW6CWnbGLp",
        "outputId": "d950c103-7aef-417a-9a9b-08c7ac014fde"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    0.865937\n",
              "1    0.134063\n",
              "Name: label_num, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Split the Dataset into Train and Test**"
      ],
      "metadata": {
        "id": "x1Egrnbgb5eV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# split train dataset into train, validation and test sets\n",
        "train_text, temp_text, train_labels, temp_labels = train_test_split(df['message'], df['label_num'],\n",
        "                                                                    random_state=2018,\n",
        "                                                                    test_size=0.3,\n",
        "                                                                    stratify=df['label_num'])\n",
        "\n",
        "\n",
        "val_text, test_text, val_labels, test_labels = train_test_split(temp_text, temp_labels,\n",
        "                                                                random_state=2018,\n",
        "                                                                test_size=0.5,\n",
        "                                                                stratify=temp_labels)"
      ],
      "metadata": {
        "id": "In6FM02eZbrt"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Import BERT-base-uncased**\n",
        "\n",
        "The BertTokenizerFast.from_pretrained() function helps to load the BERT tokenizer and is responsible for converting input text into tokens that BERT can understand. To handle the lowercase text, the ‘Bert-base-uncased’ tokenizer is used, which is aligned with the ‘Bert-base-uncased’ pre-trained model."
      ],
      "metadata": {
        "id": "n_RrX8fQcYaR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# import BERT-base pretrained model\n",
        "bert = AutoModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "# Load the BERT tokenizer\n",
        "tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177,
          "referenced_widgets": [
            "b9eb4eed50174e4fb5e8cdd8e342f8a4",
            "2adb9349be15429097ae3c6983283975",
            "3115c681901742479247ac8cc319f6ab",
            "60e6cfce53ef4410aa0c20bbd135ba98",
            "c8567b3582ad435b9af1ffa24147fd22",
            "0473282cc9a140a0bb28aa252aaff0a0",
            "f0d562b47bd6438196e15394e0a28b67",
            "73b4a2d35e4648aeab698ea47b650a48",
            "922665774c16451cba89e0900fcee46f",
            "9aa27d05552f493682ee651654c2c4d3",
            "a09d67e066c3426c99d3882d45e8461d",
            "3933ce313e0e4dcfb3e3fe504b000cdd",
            "9067620e3045437fbc29fc9dc329bad8",
            "a79d95cb6ebd4253b05575a0d3c58f19",
            "7588b9d98a8f4b858f88015eb7aa12fb",
            "c013bf2759384b0db966eaf7d717ef9f",
            "81bff108ae3b42c7b508b6bff825bcdd",
            "05bb691b372e4f48a6547446e1ece474",
            "ce4ec863ef73435cb00eae48180722c7",
            "e7c6c2b4c6f2491385a504d1001e5cb8",
            "384ae7ad4827413593545c74ef9b7cb0",
            "6b9fc4a964624468a5c6c5e1429b15c2",
            "63b8f71907f04f5499bbe72594495cb3",
            "8b9cbc9f026b450286357e290ccf3fb8",
            "6ce6b15bd00543839877150a17503758",
            "a76c79f5d4724c89bf2da445a5a8699a",
            "17ef4af4a4ba4d80930e470afb314424",
            "56c1292f202c4ce2a22685cab6f8cd34",
            "ad0bd474529249a7a9a1dc9d487c1967",
            "14a9a8a9e2cc4064b3ba49c68524bd42",
            "941e60a745c043bcbd3e1ef8e5979d29",
            "c9178f51e44a4d00a1f8241b0d565c40",
            "09599fc3e6074e24acc0732ad4150909",
            "673f49a7622f41ebb236f8199214ca7a",
            "8d66c4ccd38348d4818ee161d3bc36ba",
            "4f9bcd7f8a38449f87f918beab95bea6",
            "aeece01d7cef4cbf8c5f86bfc94bb00e",
            "f1c2476ad408400e8b1014c9a7285fb2",
            "5efa4a88decb49419ed1047e37726046",
            "2dc0aee3fa844de38623e48856e76c16",
            "4554f8e64db54c4c8408af216bfb7319",
            "63d879b9421b48d38ac225b12b4d60a6",
            "53ef815d80914418aadd0158040b2835",
            "d7e2ad77963a47d6a8442109d458fdcf",
            "4be0f98adbc546778cc0e5b1cc660566",
            "9a403deb50d4439db92a109720c13ead",
            "502def7f14ed4eb9811affbc4a9e2dfd",
            "e287f2a6a4c749fca0fd23ccc6a62b59",
            "0a04a83d14ee488fa13319ae7f3f6c72",
            "7319ebc3c5c44cb9b0c892495faf480a",
            "d511198999a7446da1834022c8408f94",
            "dd59efa48997494ab82b15d324843811",
            "71826ce129b549a4b7364246a2230b3f",
            "2a8ac884ba2148218c7bc06c1541337d",
            "a98db6dc9d5e442e9f99054f3e875bff"
          ]
        },
        "id": "hbvWjupLaDB2",
        "outputId": "52d00abd-4cad-4798-e4e9-082410adb197"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b9eb4eed50174e4fb5e8cdd8e342f8a4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading model.safetensors:   0%|          | 0.00/440M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3933ce313e0e4dcfb3e3fe504b000cdd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "63b8f71907f04f5499bbe72594495cb3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "673f49a7622f41ebb236f8199214ca7a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4be0f98adbc546778cc0e5b1cc660566"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# get length of all the messages in the train set\n",
        "seq_len = [len(i.split()) for i in train_text]\n",
        "\n",
        "pd.Series(seq_len).hist(bins = 30)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 447
        },
        "id": "mS_DxdWJaMgv",
        "outputId": "74d4c77e-a19a-4e25-ce68-9e63f568ecf0"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Axes: >"
            ]
          },
          "metadata": {},
          "execution_count": 11
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAGdCAYAAAAIbpn/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAtsUlEQVR4nO3dfXRU1aH+8WcCkwkgkxgoSaYGjNYqyqtQYnyrlZCAVFG51WhujS0XWkysmFYx/QFCfAkEixSkUO9S0CWodV1FRYoZQYlKDBDMVRApeql4C5PcGkOAlGRIzu+PWTlxDEJeJkx2+H7WYpU5Z589+zycxKdnMhOHZVmWAAAADBQR7gUAAAC0F0UGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGCsnuFeQGdpbGzUgQMH1LdvXzkcjnAvBwAAtIJlWTp8+LA8Ho8iIk59v6XbFpkDBw4oMTEx3MsAAADt8OWXX+qcc8455bhuW2T69u0rKRCE2+3u8Hx+v19FRUVKS0uT0+ns8HymIocAcmhGFgHkEEAOzcgioK051NTUKDEx0f7v+Kl02yLT9HKS2+0OWZHp3bu33G73GX9BkgM5fBNZBJBDADk0I4uA9ubQ2h8L4Yd9AQCAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIzVM9wLONOc+8Ab7T727/MnhnAlAACYjzsyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACM1eYiU1xcrOuvv14ej0cOh0Nr16619/n9fs2cOVNDhw5Vnz595PF4dMcdd+jAgQNBc1RVVSkzM1Nut1sxMTGaMmWKjhw5EjTmo48+0lVXXaWoqCglJiaqsLCwfWcIAAC6rTYXmaNHj2r48OFatmxZi321tbXasWOHZs+erR07dujll1/Wnj17dMMNNwSNy8zM1K5du+T1erVu3ToVFxdr2rRp9v6amhqlpaVp0KBBKisr08KFCzV37lw9+eST7ThFAADQXbX5c2QmTJigCRMmnHBfdHS0vF5v0LYnnnhCY8aM0f79+zVw4EDt3r1bGzZs0LZt2zR69GhJ0tKlS3Xdddfpsccek8fj0erVq1VfX6+nn35akZGRuuSSS1ReXq5FixYFFR4AAHBm6/QPxDt06JAcDodiYmIkSSUlJYqJibFLjCSlpqYqIiJCpaWluummm1RSUqKrr75akZGR9pj09HQtWLBAX3/9tc4+++wWz1NXV6e6ujr7cU1NjaTAy11+v7/D59E0R0fncvWwOryGcApVDqYjh2ZkEUAOAeTQjCwC2ppDW/Pq1CJz7NgxzZw5U7fddpvcbrckyefzacCAAcGL6NlTsbGx8vl89pikpKSgMXFxcfa+ExWZgoICzZs3r8X2oqIi9e7dOyTnI6nFHae2KhzT/mPXr1/foecOpY7m0F2QQzOyCCCHAHJoRhYBrc2htra2TfN2WpHx+/265ZZbZFmWli9f3llPY8vLy1Nubq79uKamRomJiUpLS7NLVEf4/X55vV6NGzdOTqez3fMMmftmu4/dOTe93ceGSqhyMB05NCOLAHIIIIdmZBHQ1hyaXlFprU4pMk0l5osvvtCmTZuCikR8fLwqKyuDxh8/flxVVVWKj4+3x1RUVASNaXrcNObbXC6XXC5Xi+1OpzOkF1BH56trcHToubuKUOdqKnJoRhYB5BBADs3IIqC1ObQ1q5B/jkxTidm7d6/eeust9evXL2h/SkqKqqurVVZWZm/btGmTGhsblZycbI8pLi4Oep3M6/XqwgsvPOHLSgAA4MzU5iJz5MgRlZeXq7y8XJK0b98+lZeXa//+/fL7/fq3f/s3bd++XatXr1ZDQ4N8Pp98Pp/q6+slSYMHD9b48eM1depUbd26Ve+//75ycnKUkZEhj8cjSbr99tsVGRmpKVOmaNeuXXrxxRf1xz/+MeilIwAAgDa/tLR9+3b95Cc/sR83lYusrCzNnTtXr732miRpxIgRQce9/fbbuuaaayRJq1evVk5OjsaOHauIiAhNnjxZS5YsscdGR0erqKhI2dnZGjVqlPr37685c+bw1msAABCkzUXmmmuukWV991uIT7avSWxsrNasWXPSMcOGDdO7777b1uUBAIAzCL9rCQAAGIsiAwAAjEWRAQAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjEWRAQAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFhtLjLFxcW6/vrr5fF45HA4tHbt2qD9lmVpzpw5SkhIUK9evZSamqq9e/cGjamqqlJmZqbcbrdiYmI0ZcoUHTlyJGjMRx99pKuuukpRUVFKTExUYWFh288OAAB0a20uMkePHtXw4cO1bNmyE+4vLCzUkiVLtGLFCpWWlqpPnz5KT0/XsWPH7DGZmZnatWuXvF6v1q1bp+LiYk2bNs3eX1NTo7S0NA0aNEhlZWVauHCh5s6dqyeffLIdpwgAALqrnm09YMKECZowYcIJ91mWpcWLF2vWrFmaNGmSJOnZZ59VXFyc1q5dq4yMDO3evVsbNmzQtm3bNHr0aEnS0qVLdd111+mxxx6Tx+PR6tWrVV9fr6efflqRkZG65JJLVF5erkWLFgUVHgAAcGZrc5E5mX379snn8yk1NdXeFh0dreTkZJWUlCgjI0MlJSWKiYmxS4wkpaamKiIiQqWlpbrppptUUlKiq6++WpGRkfaY9PR0LViwQF9//bXOPvvsFs9dV1enuro6+3FNTY0kye/3y+/3d/jcmubo6FyuHlaH1xBOocrBdOTQjCwCyCGAHJqRRUBbc2hrXiEtMj6fT5IUFxcXtD0uLs7e5/P5NGDAgOBF9Oyp2NjYoDFJSUkt5mjad6IiU1BQoHnz5rXYXlRUpN69e7fzjFryer0dOr5wTPuPXb9+fYeeO5Q6mkN3QQ7NyCKAHALIoRlZBLQ2h9ra2jbNG9IiE055eXnKzc21H9fU1CgxMVFpaWlyu90dnt/v98vr9WrcuHFyOp3tnmfI3DfbfezOuentPjZUQpWD6cihGVkEkEMAOTQji4C25tD0ikprhbTIxMfHS5IqKiqUkJBgb6+oqNCIESPsMZWVlUHHHT9+XFVVVfbx8fHxqqioCBrT9LhpzLe5XC65XK4W251OZ0gvoI7OV9fg6NBzdxWhztVU5NCMLALIIYAcmpFFQGtzaGtWIf0cmaSkJMXHx2vjxo32tpqaGpWWliolJUWSlJKSourqapWVldljNm3apMbGRiUnJ9tjiouLg14n83q9uvDCC0/4shIAADgztbnIHDlyROXl5SovL5cU+AHf8vJy7d+/Xw6HQzNmzNDDDz+s1157TR9//LHuuOMOeTwe3XjjjZKkwYMHa/z48Zo6daq2bt2q999/Xzk5OcrIyJDH45Ek3X777YqMjNSUKVO0a9cuvfjii/rjH/8Y9NIRAABAm19a2r59u37yk5/Yj5vKRVZWllatWqX7779fR48e1bRp01RdXa0rr7xSGzZsUFRUlH3M6tWrlZOTo7FjxyoiIkKTJ0/WkiVL7P3R0dEqKipSdna2Ro0apf79+2vOnDm89RoAAARpc5G55pprZFnf/RZih8Oh/Px85efnf+eY2NhYrVmz5qTPM2zYML377rttXR4AADiD8LuWAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjEWRAQAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjBXyItPQ0KDZs2crKSlJvXr10vnnn6+HHnpIlmXZYyzL0pw5c5SQkKBevXopNTVVe/fuDZqnqqpKmZmZcrvdiomJ0ZQpU3TkyJFQLxcAABgs5EVmwYIFWr58uZ544gnt3r1bCxYsUGFhoZYuXWqPKSws1JIlS7RixQqVlpaqT58+Sk9P17Fjx+wxmZmZ2rVrl7xer9atW6fi4mJNmzYt1MsFAAAG6xnqCbds2aJJkyZp4sSJkqRzzz1Xzz//vLZu3SopcDdm8eLFmjVrliZNmiRJevbZZxUXF6e1a9cqIyNDu3fv1oYNG7Rt2zaNHj1akrR06VJdd911euyxx+TxeEK9bAAAYKCQF5nLL79cTz75pP72t7/phz/8of77v/9b7733nhYtWiRJ2rdvn3w+n1JTU+1joqOjlZycrJKSEmVkZKikpEQxMTF2iZGk1NRURUREqLS0VDfddFOL562rq1NdXZ39uKamRpLk9/vl9/s7fF5Nc3R0LlcP69SDTrGGcApVDqYjh2ZkEUAOAeTQjCwC2ppDW/MKeZF54IEHVFNTo4suukg9evRQQ0ODHnnkEWVmZkqSfD6fJCkuLi7ouLi4OHufz+fTgAEDghfas6diY2PtMd9WUFCgefPmtdheVFSk3r17d/i8mni93g4dXzim/ceuX7++Q88dSh3Nobsgh2ZkEUAOAeTQjCwCWptDbW1tm+YNeZH5y1/+otWrV2vNmjW65JJLVF5erhkzZsjj8SgrKyvUT2fLy8tTbm6u/bimpkaJiYlKS0uT2+3u8Px+v19er1fjxo2T0+ls9zxD5r7Z7mN3zk1v97GhEqocTEcOzcgigBwCyKEZWQS0NYemV1RaK+RF5r777tMDDzygjIwMSdLQoUP1xRdfqKCgQFlZWYqPj5ckVVRUKCEhwT6uoqJCI0aMkCTFx8ersrIyaN7jx4+rqqrKPv7bXC6XXC5Xi+1OpzOkF1BH56trcHToubuKUOdqKnJoRhYB5BBADs3IIqC1ObQ1q5C/a6m2tlYREcHT9ujRQ42NjZKkpKQkxcfHa+PGjfb+mpoalZaWKiUlRZKUkpKi6upqlZWV2WM2bdqkxsZGJScnh3rJAADAUCG/I3P99dfrkUce0cCBA3XJJZfoww8/1KJFi/TLX/5SkuRwODRjxgw9/PDDuuCCC5SUlKTZs2fL4/HoxhtvlCQNHjxY48eP19SpU7VixQr5/X7l5OQoIyODdywBAABbyIvM0qVLNXv2bN11112qrKyUx+PRr371K82ZM8cec//99+vo0aOaNm2aqqurdeWVV2rDhg2Kioqyx6xevVo5OTkaO3asIiIiNHnyZC1ZsiTUywUAAAYLeZHp27evFi9erMWLF3/nGIfDofz8fOXn53/nmNjYWK1ZsybUywMAAN0Iv2sJAAAYK+R3ZM4E5z7wRriXAAAAxB0ZAABgMIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjEWRAQAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjNUpReYf//iH/v3f/139+vVTr169NHToUG3fvt3eb1mW5syZo4SEBPXq1Uupqanau3dv0BxVVVXKzMyU2+1WTEyMpkyZoiNHjnTGcgEAgKFCXmS+/vprXXHFFXI6nfrrX/+qTz75RH/4wx909tln22MKCwu1ZMkSrVixQqWlperTp4/S09N17Ngxe0xmZqZ27dolr9erdevWqbi4WNOmTQv1cgEAgMF6hnrCBQsWKDExUStXrrS3JSUl2X+3LEuLFy/WrFmzNGnSJEnSs88+q7i4OK1du1YZGRnavXu3NmzYoG3btmn06NGSpKVLl+q6667TY489Jo/HE+plAwAAA4W8yLz22mtKT0/Xz372M23evFnf//73ddddd2nq1KmSpH379snn8yk1NdU+Jjo6WsnJySopKVFGRoZKSkoUExNjlxhJSk1NVUREhEpLS3XTTTe1eN66ujrV1dXZj2tqaiRJfr9ffr+/w+fVNIff75erh9Xh+TqyhnD6Zg5nMnJoRhYB5BBADs3IIqCtObQ1r5AXmf/5n//R8uXLlZubq9///vfatm2bfvOb3ygyMlJZWVny+XySpLi4uKDj4uLi7H0+n08DBgwIXmjPnoqNjbXHfFtBQYHmzZvXYntRUZF69+4dilOTJHm9XhWOCdl0bbJ+/frwPPEJeL3ecC+hSyCHZmQRQA4B5NCMLAJam0NtbW2b5g15kWlsbNTo0aP16KOPSpJGjhypnTt3asWKFcrKygr109ny8vKUm5trP66pqVFiYqLS0tLkdrs7PL/f75fX69W4ceM08pFNHZ6vPXbOTQ/L837TN3NwOp3hXk7YkEMzsggghwByaEYWAW3NoekVldYKeZFJSEjQxRdfHLRt8ODB+q//+i9JUnx8vCSpoqJCCQkJ9piKigqNGDHCHlNZWRk0x/Hjx1VVVWUf/20ul0sul6vFdqfTGdILyOl0qq7BEbL52vrcXUWoczUVOTQjiwByCCCHZmQR0Noc2ppVyN+1dMUVV2jPnj1B2/72t79p0KBBkgI/+BsfH6+NGzfa+2tqalRaWqqUlBRJUkpKiqqrq1VWVmaP2bRpkxobG5WcnBzqJQMAAEOF/I7Mvffeq8svv1yPPvqobrnlFm3dulVPPvmknnzySUmSw+HQjBkz9PDDD+uCCy5QUlKSZs+eLY/HoxtvvFFS4A7O+PHjNXXqVK1YsUJ+v185OTnKyMjgHUsAAMAW8iLzox/9SK+88ory8vKUn5+vpKQkLV68WJmZmfaY+++/X0ePHtW0adNUXV2tK6+8Uhs2bFBUVJQ9ZvXq1crJydHYsWMVERGhyZMna8mSJaFeLgAAMFjIi4wk/fSnP9VPf/rT79zvcDiUn5+v/Pz87xwTGxurNWvWdMbyAABAN8HvWgIAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjNUpH4iH7ufcB96QJLl6WCocIw2Z+2arf3nm3+dP7MylAQDOYNyRAQAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYi8+RMUjTZ7m0B5/lAgDojrgjAwAAjEWRAQAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMxSf7niE68qnAAAB0VdyRAQAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLE6vcjMnz9fDodDM2bMsLcdO3ZM2dnZ6tevn8466yxNnjxZFRUVQcft379fEydOVO/evTVgwADdd999On78eGcvFwAAGKRTi8y2bdv05z//WcOGDQvafu+99+r111/XSy+9pM2bN+vAgQO6+eab7f0NDQ2aOHGi6uvrtWXLFj3zzDNatWqV5syZ05nLBQAAhum0InPkyBFlZmbqP//zP3X22Wfb2w8dOqSnnnpKixYt0rXXXqtRo0Zp5cqV2rJliz744ANJUlFRkT755BM999xzGjFihCZMmKCHHnpIy5YtU319fWctGQAAGKZnZ02cnZ2tiRMnKjU1VQ8//LC9vaysTH6/X6mpqfa2iy66SAMHDlRJSYkuu+wylZSUaOjQoYqLi7PHpKena/r06dq1a5dGjhzZ4vnq6upUV1dnP66pqZEk+f1++f3+Dp9P0xx+v1+uHlaH5zOVK8IK+t/WCEX+Xc03r4czHVkEkEMAOTQji4C25tDWvDqlyLzwwgvasWOHtm3b1mKfz+dTZGSkYmJigrbHxcXJ5/PZY75ZYpr2N+07kYKCAs2bN6/F9qKiIvXu3bs9p3FCXq9XhWNCNp2xHhrd2Oqx69ev78SVhJfX6w33EroMsggghwByaEYWAa3Noba2tk3zhrzIfPnll7rnnnvk9XoVFRUV6um/U15ennJzc+3HNTU1SkxMVFpamtxud4fn9/v98nq9GjdunEY+sqnD85nKFWHpodGNmr09QnWNjlYds3Nueiev6vT75vXgdDrDvZywIosAcgggh2ZkEdDWHJpeUWmtkBeZsrIyVVZW6tJLL7W3NTQ0qLi4WE888YTefPNN1dfXq7q6OuiuTEVFheLj4yVJ8fHx2rp1a9C8Te9qahrzbS6XSy6Xq8V2p9MZ0gvI6XSqrqF1/wHvzuoaHa3OoTt/AYf6+jIZWQSQQwA5NCOLgNbm0NasQv7DvmPHjtXHH3+s8vJy+8/o0aOVmZlp/93pdGrjxo32MXv27NH+/fuVkpIiSUpJSdHHH3+syspKe4zX65Xb7dbFF18c6iUDAABDhfyOTN++fTVkyJCgbX369FG/fv3s7VOmTFFubq5iY2Pldrt19913KyUlRZdddpkkKS0tTRdffLF+/vOfq7CwUD6fT7NmzVJ2dvYJ77oAAIAzU6e9a+lkHn/8cUVERGjy5Mmqq6tTenq6/vSnP9n7e/TooXXr1mn69OlKSUlRnz59lJWVpfz8/HAsFwAAdFGnpci88847QY+joqK0bNkyLVu27DuPGTRoULd+twsAAOg4ftcSAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjEWRAQAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjEWRAQAAxqLIAAAAY1FkAACAsXqGesKCggK9/PLL+vTTT9WrVy9dfvnlWrBggS688EJ7zLFjx/Tb3/5WL7zwgurq6pSenq4//elPiouLs8fs379f06dP19tvv62zzjpLWVlZKigoUM+eIV8yOtm5D7zR7mP/Pn9iCFcCAOhuQn5HZvPmzcrOztYHH3wgr9crv9+vtLQ0HT161B5z77336vXXX9dLL72kzZs368CBA7r55pvt/Q0NDZo4caLq6+u1ZcsWPfPMM1q1apXmzJkT6uUCAACDhfz2xoYNG4Ier1q1SgMGDFBZWZmuvvpqHTp0SE899ZTWrFmja6+9VpK0cuVKDR48WB988IEuu+wyFRUV6ZNPPtFbb72luLg4jRgxQg899JBmzpypuXPnKjIyMtTLBgAABur012kOHTokSYqNjZUklZWVye/3KzU11R5z0UUXaeDAgSopKdFll12mkpISDR06NOilpvT0dE2fPl27du3SyJEjWzxPXV2d6urq7Mc1NTWSJL/fL7/f3+HzaJrD7/fL1cPq8HymckVYQf/b2ULxb9cZvnk9nOnIIoAcAsihGVkEtDWHtubVqUWmsbFRM2bM0BVXXKEhQ4ZIknw+nyIjIxUTExM0Ni4uTj6fzx7zzRLTtL9p34kUFBRo3rx5LbYXFRWpd+/eHT0Vm9frVeGYkE1nrIdGN56W51m/fv1peZ728nq94V5Cl0EWAeQQQA7NyCKgtTnU1ta2ad5OLTLZ2dnauXOn3nvvvc58GklSXl6ecnNz7cc1NTVKTExUWlqa3G53h+f3+/3yer0aN26cRj6yqcPzmcoVYemh0Y2avT1CdY2OTn++nXPTO/052uOb14PT6Qz3csKKLALIIYAcmpFFQFtzaHpFpbU6rcjk5ORo3bp1Ki4u1jnnnGNvj4+PV319vaqrq4PuylRUVCg+Pt4es3Xr1qD5Kioq7H0n4nK55HK5Wmx3Op0hvYCcTqfqGjr/P+BdXV2j47Tk0NW/+EN9fZmMLALIIYAcmpFFQGtzaGtWIX/XkmVZysnJ0SuvvKJNmzYpKSkpaP+oUaPkdDq1ceNGe9uePXu0f/9+paSkSJJSUlL08ccfq7Ky0h7j9Xrldrt18cUXh3rJAADAUCG/I5Odna01a9bo1VdfVd++fe2faYmOjlavXr0UHR2tKVOmKDc3V7GxsXK73br77ruVkpKiyy67TJKUlpamiy++WD//+c9VWFgon8+nWbNmKTs7+4R3XQAAwJkp5EVm+fLlkqRrrrkmaPvKlSt15513SpIef/xxRUREaPLkyUEfiNekR48eWrdunaZPn66UlBT16dNHWVlZys/PD/VyAQCAwUJeZCzr1G/LjYqK0rJly7Rs2bLvHDNo0KAu/44VAAAQXvyuJQAAYCyKDAAAMBZFBgAAGIsiAwAAjEWRAQAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjEWRAQAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgrJ7hXgBwMuc+8Ea7j/37/IkhXAkAoCvijgwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGItfUYBui19vAADdH3dkAACAsbr0HZlly5Zp4cKF8vl8Gj58uJYuXaoxY8aEe1lAp+EuEgC0TZe9I/Piiy8qNzdXDz74oHbs2KHhw4crPT1dlZWV4V4aAADoIrrsHZlFixZp6tSp+sUvfiFJWrFihd544w09/fTTeuCBB8K8OuC7deSuCgCgbbpkkamvr1dZWZny8vLsbREREUpNTVVJSckJj6mrq1NdXZ39+NChQ5Kkqqoq+f3+Dq/J7/ertrZWX331lXoeP9rh+UzVs9FSbW2jevoj1NDoCPdyOs0PfveXk+53RViaNbJRI/7fy6r7Vg7h+qL66quv2n1scsHGdh97sixOpTRvbLuftyM6cr7fteZvfo9wOp3tnt905NCMLALamsPhw4clSZZltWr+Lllk/vnPf6qhoUFxcXFB2+Pi4vTpp5+e8JiCggLNmzevxfakpKROWeOZ7PZwL6CL6Go59P9D+J67vVmEc83tZeKaARMdPnxY0dHRpxzXJYtMe+Tl5Sk3N9d+3NjYqKqqKvXr108OR8fvHNTU1CgxMVFffvml3G53h+czFTkEkEMzsggghwByaEYWAW3NwbIsHT58WB6Pp1Xzd8ki079/f/Xo0UMVFRVB2ysqKhQfH3/CY1wul1wuV9C2mJiYkK/N7Xaf0RdkE3IIIIdmZBFADgHk0IwsAtqSQ2vuxDTpku9aioyM1KhRo7RxY/Pr2I2Njdq4caNSUlLCuDIAANCVdMk7MpKUm5urrKwsjR49WmPGjNHixYt19OhR+11MAAAAXbbI3Hrrrfq///s/zZkzRz6fTyNGjNCGDRta/ADw6eJyufTggw+2ePnqTEMOAeTQjCwCyCGAHJqRRUBn5+CwWvv+JgAAgC6mS/6MDAAAQGtQZAAAgLEoMgAAwFgUGQAAYCyKTCssW7ZM5557rqKiopScnKytW7eGe0mdqqCgQD/60Y/Ut29fDRgwQDfeeKP27NkTNOaaa66Rw+EI+vPrX/86TCvuPHPnzm1xnhdddJG9/9ixY8rOzla/fv101llnafLkyS0+yLE7OPfcc1vk4HA4lJ2dLan7Xg/FxcW6/vrr5fF45HA4tHbt2qD9lmVpzpw5SkhIUK9evZSamqq9e/cGjamqqlJmZqbcbrdiYmI0ZcoUHTly5DSeRWicLAu/36+ZM2dq6NCh6tOnjzwej+644w4dOHAgaI4TXUfz588/zWfSMae6Ju68884W5zh+/PigMd3hmjhVDif6fuFwOLRw4UJ7TKiuB4rMKbz44ovKzc3Vgw8+qB07dmj48OFKT09XZWVluJfWaTZv3qzs7Gx98MEH8nq98vv9SktL09Gjwb8sc+rUqTp48KD9p7CwMEwr7lyXXHJJ0Hm+99579r57771Xr7/+ul566SVt3rxZBw4c0M033xzG1XaObdu2BWXg9XolST/72c/sMd3xejh69KiGDx+uZcuWnXB/YWGhlixZohUrVqi0tFR9+vRRenq6jh07Zo/JzMzUrl275PV6tW7dOhUXF2vatGmn6xRC5mRZ1NbWaseOHZo9e7Z27Nihl19+WXv27NENN9zQYmx+fn7QdXL33XefjuWHzKmuCUkaP3580Dk+//zzQfu7wzVxqhy+ef4HDx7U008/LYfDocmTJweNC8n1YOGkxowZY2VnZ9uPGxoaLI/HYxUUFIRxVadXZWWlJcnavHmzve3HP/6xdc8994RvUafJgw8+aA0fPvyE+6qrqy2n02m99NJL9rbdu3dbkqySkpLTtMLwuOeee6zzzz/famxstCzrzLgeJFmvvPKK/bixsdGKj4+3Fi5caG+rrq62XC6X9fzzz1uWZVmffPKJJcnatm2bPeavf/2r5XA4rH/84x+nbe2h9u0sTmTr1q2WJOuLL76wtw0aNMh6/PHHO3dxp9GJcsjKyrImTZr0ncd0x2uiNdfDpEmTrGuvvTZoW6iuB+7InER9fb3KysqUmppqb4uIiFBqaqpKSkrCuLLT69ChQ5Kk2NjYoO2rV69W//79NWTIEOXl5am2tjYcy+t0e/fulcfj0XnnnafMzEzt379fklRWVia/3x90fVx00UUaOHBgt74+6uvr9dxzz+mXv/xl0C9kPVOuhyb79u2Tz+cL+vePjo5WcnKy/e9fUlKimJgYjR492h6TmpqqiIgIlZaWnvY1n06HDh2Sw+Fo8Tvv5s+fr379+mnkyJFauHChjh8/Hp4FdqJ33nlHAwYM0IUXXqjp06frq6++svediddERUWF3njjDU2ZMqXFvlBcD132k327gn/+859qaGho8WnCcXFx+vTTT8O0qtOrsbFRM2bM0BVXXKEhQ4bY22+//XYNGjRIHo9HH330kWbOnKk9e/bo5ZdfDuNqQy85OVmrVq3ShRdeqIMHD2revHm66qqrtHPnTvl8PkVGRrb4Rh0XFyefzxeeBZ8Ga9euVXV1te68805725lyPXxT07/xib4/NO3z+XwaMGBA0P6ePXsqNja2W18jx44d08yZM3XbbbcF/ZLA3/zmN7r00ksVGxurLVu2KC8vTwcPHtSiRYvCuNrQGj9+vG6++WYlJSXp888/1+9//3tNmDBBJSUl6tGjxxl5TTzzzDPq27dvi5fdQ3U9UGRwUtnZ2dq5c2fQz4VICno9d+jQoUpISNDYsWP1+eef6/zzzz/dy+w0EyZMsP8+bNgwJScna9CgQfrLX/6iXr16hXFl4fPUU09pwoQJ8ng89rYz5XrAqfn9ft1yyy2yLEvLly8P2pebm2v/fdiwYYqMjNSvfvUrFRQUdJuP8c/IyLD/PnToUA0bNkznn3++3nnnHY0dOzaMKwufp59+WpmZmYqKigraHqrrgZeWTqJ///7q0aNHi3ehVFRUKD4+PkyrOn1ycnK0bt06vf322zrnnHNOOjY5OVmS9Nlnn52OpYVNTEyMfvjDH+qzzz5TfHy86uvrVV1dHTSmO18fX3zxhd566y39x3/8x0nHnQnXQ9O/8cm+P8THx7d4Y8Dx48dVVVXVLa+RphLzxRdfyOv1Bt2NOZHk5GQdP35cf//730/PAsPgvPPOU//+/e2vhTPtmnj33Xe1Z8+eU37PkNp/PVBkTiIyMlKjRo3Sxo0b7W2NjY3auHGjUlJSwriyzmVZlnJycvTKK69o06ZNSkpKOuUx5eXlkqSEhIROXl14HTlyRJ9//rkSEhI0atQoOZ3OoOtjz5492r9/f7e9PlauXKkBAwZo4sSJJx13JlwPSUlJio+PD/r3r6mpUWlpqf3vn5KSourqapWVldljNm3apMbGRrvsdRdNJWbv3r1666231K9fv1MeU15eroiIiBYvtXQn//u//6uvvvrK/lo4k64JKXAHd9SoURo+fPgpx7b7eujwjwt3cy+88ILlcrmsVatWWZ988ok1bdo0KyYmxvL5fOFeWqeZPn26FR0dbb3zzjvWwYMH7T+1tbWWZVnWZ599ZuXn51vbt2+39u3bZ7366qvWeeedZ1199dVhXnno/fa3v7Xeeecda9++fdb7779vpaamWv3797cqKysty7KsX//619bAgQOtTZs2Wdu3b7dSUlKslJSUMK+6czQ0NFgDBw60Zs6cGbS9O18Phw8ftj788EPrww8/tCRZixYtsj788EP7nTjz58+3YmJirFdffdX66KOPrEmTJllJSUnWv/71L3uO8ePHWyNHjrRKS0ut9957z7rgggus2267LVyn1G4ny6K+vt664YYbrHPOOccqLy8P+r5RV1dnWZZlbdmyxXr88cet8vJy6/PPP7eee+4563vf+551xx13hPnM2uZkORw+fNj63e9+Z5WUlFj79u2z3nrrLevSSy+1LrjgAuvYsWP2HN3hmjjV14ZlWdahQ4es3r17W8uXL29xfCivB4pMKyxdutQaOHCgFRkZaY0ZM8b64IMPwr2kTiXphH9WrlxpWZZl7d+/37r66qut2NhYy+VyWT/4wQ+s++67zzp06FB4F94Jbr31VishIcGKjIy0vv/971u33nqr9dlnn9n7//Wvf1l33XWXdfbZZ1u9e/e2brrpJuvgwYNhXHHnefPNNy1J1p49e4K2d+fr4e233z7h10JWVpZlWYG3YM+ePduKi4uzXC6XNXbs2Bb5fPXVV9Ztt91mnXXWWZbb7bZ+8YtfWIcPHw7D2XTMybLYt2/fd37fePvtty3LsqyysjIrOTnZio6OtqKioqzBgwdbjz76aNB/4E1wshxqa2uttLQ063vf+57ldDqtQYMGWVOnTm3xf3y7wzVxqq8Ny7KsP//5z1avXr2s6urqFseH8npwWJZlte0eDgAAQNfAz8gAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYKz/D2yq2avdAKOAAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Tokenizing and encoding the sequences**\n",
        "\n",
        "\n",
        "The BERT tokenizer can be used to tokenize and encode the sequences in the training, validation, and test sets. With the help of the tokenizer.batch_encode_plus() function, we transform the text sequences into numerical tokens.\n",
        "\n",
        "To maintain uniformity in sequence length, a maximum length of 25 is kept for each set. The pad_to_max_length=True parameter is set, and we pad the sequences accordingly. Keeping the truncation=True, we truncate the sequences larger than the specified maximum length."
      ],
      "metadata": {
        "id": "iReG6gLDdCaC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# tokenize and encode sequences in the training set\n",
        "tokens_train = tokenizer.batch_encode_plus(\n",
        "    train_text.tolist(),\n",
        "    max_length = 25,\n",
        "    pad_to_max_length=True,\n",
        "    truncation=True\n",
        ")\n",
        "\n",
        "# tokenize and encode sequences in the validation set\n",
        "tokens_val = tokenizer.batch_encode_plus(\n",
        "    val_text.tolist(),\n",
        "    max_length = 25,\n",
        "    pad_to_max_length=True,\n",
        "    truncation=True\n",
        ")\n",
        "\n",
        "# tokenize and encode sequences in the test set\n",
        "tokens_test = tokenizer.batch_encode_plus(\n",
        "    test_text.tolist(),\n",
        "    max_length = 25,\n",
        "    pad_to_max_length=True,\n",
        "    truncation=True\n",
        ")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S0en0xFEaTnc",
        "outputId": "743a9d8f-cc4b-479f-d380-f4cb84086054"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2606: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Convert the tokenized sequences and corresponding labels into tensors**\n",
        "\n",
        "\n",
        "The \"torch. tensor()\" function is used for creating tensors from the tokenized sequences and labels.\n",
        "\n",
        "\n",
        "For each of the training, validation, and test sets, we convert the tokenized input sequences to tensors using \"torch. tensor(tokens_train['input_ids'])\". Similarly, we also convert the attention masks to tensors using a \"torch. tensor(tokens_train['attention_mask'])\". To convert the labels to tensors, we utilize a torch.tensor(train_labels.tolist()).\n",
        "We convert data to tensors for efficient computation and better compatibility with PyTorch models. It also enables further processing and training using the BERT model."
      ],
      "metadata": {
        "id": "R3rbb0_Dhnly"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_seq = torch.tensor(tokens_train['input_ids'])\n",
        "train_mask = torch.tensor(tokens_train['attention_mask'])\n",
        "train_y = torch.tensor(train_labels.tolist())\n",
        "\n",
        "val_seq = torch.tensor(tokens_val['input_ids'])\n",
        "val_mask = torch.tensor(tokens_val['attention_mask'])\n",
        "val_y = torch.tensor(val_labels.tolist())\n",
        "\n",
        "test_seq = torch.tensor(tokens_test['input_ids'])\n",
        "test_mask = torch.tensor(tokens_test['attention_mask'])\n",
        "test_y = torch.tensor(test_labels.tolist())"
      ],
      "metadata": {
        "id": "3xDS-1AcajZb"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Data Loader**\n",
        "\n",
        "\n",
        "The benefit of the TensorDataset class is to wrap the input sequences, attention masks, and labels into a single dataset object.\n",
        "\n",
        "\n",
        "The RandomSampler can be used to randomly sample the training set by ensuring diverse data representation during training. The SequentialSampler for the validation set can be deployed to sequentially test the data.\n",
        "\n",
        "\n",
        "For facilitating the efficient iteration and batching of the data during training and validation, the DataLoader can be deployed. The tool can enable the creation of iterators over the datasets with a defined batch size by streamlining the entire process."
      ],
      "metadata": {
        "id": "_8FCondzikvG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "#define a batch size\n",
        "batch_size = 32\n",
        "\n",
        "# wrap tensors\n",
        "train_data = TensorDataset(train_seq, train_mask, train_y)\n",
        "\n",
        "# sampler for sampling the data during training\n",
        "train_sampler = RandomSampler(train_data)\n",
        "\n",
        "# dataLoader for train set\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "# wrap tensors\n",
        "val_data = TensorDataset(val_seq, val_mask, val_y)\n",
        "\n",
        "# sampler for sampling the data during training\n",
        "val_sampler = SequentialSampler(val_data)\n",
        "\n",
        "# dataLoader for validation set\n",
        "val_dataloader = DataLoader(val_data, sampler = val_sampler, batch_size=batch_size)"
      ],
      "metadata": {
        "id": "JnbuSFyWihxy"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Model Architecture**\n",
        "\n",
        "\n",
        "The BERT_Arch class extends the nn.Module class for initializing the BERT model as a parameter.\n",
        "To set the parameters of the BERT model, we can ignore the gradients by making param.requires_grad = False. Only the parameters of the added layers are trained during the training process. As a consequence, the pre-trained BERT model for transfer learning has been adapted for a specific task."
      ],
      "metadata": {
        "id": "hJwM1kAOjS1g"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# freeze all the parameters\n",
        "for param in bert.parameters():\n",
        "    param.requires_grad = False"
      ],
      "metadata": {
        "id": "Eldv7rZCjQ7j"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "The present architecture comprises a dropout layer, a ReLU activation function, 2 dense layers with 768 and 512 units, respectively, and a softmax activation function. Sentence IDs and masks are taken as inputs to the forward method, followed by passing them into the BERT model for obtaining the output from the classification token. It finally applies the defined layers and activations for producing the final classification probabilities.\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "IvOdAgI7kPJM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class BERT_Arch(nn.Module):\n",
        "\n",
        "    def __init__(self, bert):\n",
        "        super(BERT_Arch, self).__init__()\n",
        "\n",
        "        self.bert = bert\n",
        "\n",
        "        # dropout layer\n",
        "        self.dropout = nn.Dropout(0.1)\n",
        "\n",
        "        # relu activation function\n",
        "        self.relu =  nn.ReLU()\n",
        "\n",
        "        # dense layer 1\n",
        "        self.fc1 = nn.Linear(768,512)\n",
        "\n",
        "        # dense layer 2 (Output layer)\n",
        "        self.fc2 = nn.Linear(512,2)\n",
        "\n",
        "        #softmax activation function\n",
        "        self.softmax = nn.LogSoftmax(dim=1)\n",
        "\n",
        "    #define the forward pass\n",
        "    def forward(self, sent_id, mask):\n",
        "\n",
        "        #pass the inputs to the model\n",
        "        _, cls_hs = self.bert(sent_id, attention_mask=mask, return_dict=False)\n",
        "\n",
        "        x = self.fc1(cls_hs)\n",
        "\n",
        "        x = self.relu(x)\n",
        "\n",
        "        x = self.dropout(x)\n",
        "\n",
        "        # output layer\n",
        "        x = self.fc2(x)\n",
        "\n",
        "        # apply softmax activation\n",
        "        x = self.softmax(x)\n",
        "\n",
        "        return x"
      ],
      "metadata": {
        "id": "LyVkKUpFj_2x"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "For initializing an instance of the BERT_Arch class with the BERT model as an argument, the pre-trained BERT model for the defined architecture has been passed to establish the BERT model as the backbone of the custom architecture."
      ],
      "metadata": {
        "id": "eVzAI7b1kC1S"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# pass the pre-trained BERT to our define architecture\n",
        "model = BERT_Arch(bert)"
      ],
      "metadata": {
        "id": "UGcroyqclPW-"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "The Hugging Face has the Transformers library. AdamW, a variant of the Adam optimizer, can be imported from here. AdamW includes weight decay regularization.\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "usefktT2p9m4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# optimizer from hugging face transformers\n",
        "from transformers import AdamW\n",
        "\n",
        "# define the optimizer\n",
        "optimizer = AdamW(model.parameters(),lr = 1e-5)\n",
        "\n",
        "from sklearn.utils.class_weight import compute_class_weight\n",
        "\n",
        "y = train_labels\n",
        "classes=np.unique(y)\n",
        "\n",
        "#compute the class weights\n",
        "class_weights = compute_class_weight('balanced', classes = classes, y = y)\n",
        "\n",
        "print('Class Weights:',class_weights)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9BfL45JjkYfD",
        "outputId": "f7621f10-4d7f-4ba1-8b8c-6db2c690c9c3"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Class Weights: [0.57743559 3.72848948]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# converting list of class weights to a tensor\n",
        "weights= torch.tensor(class_weights,dtype=torch.float)\n",
        "\n",
        "# push to GPU\n",
        "#weights = weights.to(device)\n",
        "\n",
        "# define the loss function\n",
        "cross_entropy  = nn.NLLLoss(weight=weights)\n",
        "\n",
        "# number of training epochs\n",
        "epochs = 10"
      ],
      "metadata": {
        "id": "LD2VWTVopwKa"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Fine-tuning**"
      ],
      "metadata": {
        "id": "4o6e8dNfrMCy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# function to train the model\n",
        "def train():\n",
        "\n",
        "    model.train()\n",
        "    total_loss, total_accuracy = 0, 0\n",
        "\n",
        "    # empty list to save model predictions\n",
        "    total_preds=[]\n",
        "\n",
        "    # iterate over batches\n",
        "    for step,batch in enumerate(train_dataloader):\n",
        "\n",
        "        # progress update after every 50 batches.\n",
        "        if step % 50 == 0 and not step == 0:\n",
        "            print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(train_dataloader)))\n",
        "\n",
        "        # push the batch to gpu\n",
        "        #batch = [r.to(device) for r in batch]\n",
        "\n",
        "        sent_id, mask, labels = batch\n",
        "\n",
        "        # clear previously calculated gradients\n",
        "        model.zero_grad()\n",
        "\n",
        "        # get model predictions for the current batch\n",
        "        preds = model(sent_id, mask)\n",
        "\n",
        "        # compute the loss between actual and predicted values\n",
        "        loss = cross_entropy(preds, labels)\n",
        "\n",
        "        # add on to the total loss\n",
        "        total_loss = total_loss + loss.item()\n",
        "\n",
        "        # backward pass to calculate the gradients\n",
        "        loss.backward()\n",
        "        # clip the the gradients to 1.0. It helps in preventing the exploding gradient problem\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        # update parameters\n",
        "        optimizer.step()\n",
        "\n",
        "        # model predictions are stored on GPU. So, push it to CPU\n",
        "        preds=preds.detach().cpu().numpy()\n",
        "\n",
        "    # append the model predictions\n",
        "    total_preds.append(preds)\n",
        "\n",
        "    # compute the training loss of the epoch\n",
        "    avg_loss = total_loss / len(train_dataloader)\n",
        "    # predictions are in the form of (no. of batches, size of batch, no. of classes).\n",
        "      # reshape the predictions in form of (number of samples, no. of classes)\n",
        "    total_preds  = np.concatenate(total_preds, axis=0)\n",
        "\n",
        "    #returns the loss and predictions\n",
        "    return avg_loss, total_preds"
      ],
      "metadata": {
        "id": "5hoX8vV9qWKO"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "The evaluation function evaluates the model based on the validation data, computes the validation loss by storing the model predictions, and returns the average loss and predictions. The dropout layers have been deactivated by the function, and forward passes are performed without gradient computation using torch.no_grad()."
      ],
      "metadata": {
        "id": "3FZiro-yrzG1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# function for evaluating the model\n",
        "def evaluate():\n",
        "\n",
        "    print(\"\\nEvaluating...\")\n",
        "\n",
        "    # deactivate dropout layers\n",
        "    model.eval()\n",
        "\n",
        "    total_loss, total_accuracy = 0, 0\n",
        "\n",
        "    # empty list to save the model predictions\n",
        "    total_preds = []\n",
        "\n",
        "    # iterate over batches\n",
        "    for step,batch in enumerate(val_dataloader):\n",
        "\n",
        "        # Progress update every 50 batches.\n",
        "        if step % 50 == 0 and not step == 0:\n",
        "\n",
        "            # Calculate elapsed time in minutes.\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "\n",
        "            # Report progress.\n",
        "            print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(val_dataloader)))\n",
        "\n",
        "        # push the batch to gpu\n",
        "       # batch = [t.to(device) for t in batch]\n",
        "\n",
        "        sent_id, mask, labels = batch\n",
        "\n",
        "        # deactivate autograd\n",
        "        with torch.no_grad():\n",
        "\n",
        "            # model predictions\n",
        "            preds = model(sent_id, mask)\n",
        "\n",
        "            # compute the validation loss between actual and predicted values\n",
        "            loss = cross_entropy(preds,labels)\n",
        "\n",
        "            total_loss = total_loss + loss.item()\n",
        "\n",
        "            preds = preds.detach().cpu().numpy()\n",
        "\n",
        "            total_preds.append(preds)\n",
        "\n",
        "    # compute the validation loss of the epoch\n",
        "    avg_loss = total_loss / len(val_dataloader)\n",
        "\n",
        "    # reshape the predictions in form of (number of samples, no. of classes)\n",
        "    total_preds  = np.concatenate(total_preds, axis=0)\n",
        "\n",
        "    return avg_loss, total_preds"
      ],
      "metadata": {
        "id": "-lQlycxarPLj"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Training the model**"
      ],
      "metadata": {
        "id": "fifkrqWYpdYT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# set initial loss to infinite\n",
        "best_valid_loss = float('inf')\n",
        "\n",
        "#defining epochs\n",
        "epochs = 10\n",
        "\n",
        "# empty lists to store training and validation loss of each epoch\n",
        "train_losses=[]\n",
        "valid_losses=[]\n",
        "\n",
        "#for each epoch\n",
        "for epoch in range(epochs):\n",
        "\n",
        "    print('\\n Epoch {:} / {:}'.format(epoch + 1, epochs))\n",
        "\n",
        "    #train model\n",
        "    train_loss, _ = train()\n",
        "\n",
        "    #evaluate model\n",
        "    valid_loss, _ = evaluate()\n",
        "\n",
        "    #save the best model\n",
        "    if valid_loss < best_valid_loss:\n",
        "        best_valid_loss = valid_loss\n",
        "        torch.save(model.state_dict(), 'saved_weights.pt')\n",
        "\n",
        "    # append training and validation loss\n",
        "    train_losses.append(train_loss)\n",
        "    valid_losses.append(valid_loss)\n",
        "\n",
        "    print(f'\\nTraining Loss: {train_loss:.3f}')\n",
        "    print(f'Validation Loss: {valid_loss:.3f}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iooWqWFMrhxq",
        "outputId": "2adcf4c4-bec4-44b3-97c3-973ca7a385db"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " Epoch 1 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.649\n",
            "Validation Loss: 0.627\n",
            "\n",
            " Epoch 2 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.621\n",
            "Validation Loss: 0.600\n",
            "\n",
            " Epoch 3 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.594\n",
            "Validation Loss: 0.573\n",
            "\n",
            " Epoch 4 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.576\n",
            "Validation Loss: 0.553\n",
            "\n",
            " Epoch 5 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.553\n",
            "Validation Loss: 0.529\n",
            "\n",
            " Epoch 6 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.531\n",
            "Validation Loss: 0.507\n",
            "\n",
            " Epoch 7 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.523\n",
            "Validation Loss: 0.491\n",
            "\n",
            " Epoch 8 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.493\n",
            "Validation Loss: 0.474\n",
            "\n",
            " Epoch 9 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.478\n",
            "Validation Loss: 0.453\n",
            "\n",
            " Epoch 10 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.464\n",
            "Validation Loss: 0.440\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "For loading the best model weights from the saved file  ‘saved_weights.pt’, we are using torch.load() and setting them in the model using model.load_state_dict()."
      ],
      "metadata": {
        "id": "vDaKDr79ofTF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#load weights of best model\n",
        "path = 'saved_weights.pt'\n",
        "model.load_state_dict(torch.load(path))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l7L0wwvNdsqX",
        "outputId": "edea3064-e804-4311-e520-8a0761337606"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Predictions**\n",
        "\n",
        "For making predictions on the test data using the trained model and converting the predictions to NumPy arrays. The classification metrics, including precision, recall, and F1-score, have been computed to evaluate the model’s performance using the classification_report function from scikit-learn’s metrics module."
      ],
      "metadata": {
        "id": "5wNUwgHjjTAW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# get predictions for test data\n",
        "with torch.no_grad():\n",
        "    #preds = model(test_seq.to(device), test_mask.to(device))\n",
        "    preds = model(test_seq, test_mask)\n",
        "    preds = preds.detach().cpu().numpy()\n",
        "\n",
        "\n",
        "# model's performance\n",
        "preds = np.argmax(preds, axis = 1)\n",
        "print(classification_report(test_y, preds))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3aJ7NgrFdr__",
        "outputId": "0935dc78-b6a7-43b9-f721-1e9ee6027b8c"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.90      0.93       724\n",
            "           1       0.53      0.75      0.62       112\n",
            "\n",
            "    accuracy                           0.88       836\n",
            "   macro avg       0.75      0.82      0.77       836\n",
            "weighted avg       0.90      0.88      0.89       836\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Therefore, we achieved a final prediction accuracy of 88%."
      ],
      "metadata": {
        "id": "0-V-NiIu-HLA"
      }
    }
  ]
}