MCD_BERT_EN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Cross1 MCD BERT EN",
      "provenance": [],
      "authorship_tag": "ABX9TyM2fhYAP0bc65EQWNBgDeNJ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b693c13b37a64d7289775cc1b8544ab5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_eded4a106b3e47c2ad5e61d07f93aaeb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6d0c09e222444dcbae1381a0ace60733",
              "IPY_MODEL_16410aeb46d5454587056b100582e71c"
            ]
          }
        },
        "eded4a106b3e47c2ad5e61d07f93aaeb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6d0c09e222444dcbae1381a0ace60733": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_749db83dd99c4c0b9607f9dc81e2a429",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_83e2e50b3af5464fa7078f6e6e0c158d"
          }
        },
        "16410aeb46d5454587056b100582e71c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e9c2d26361644e1e847cfffd8b2f3cca",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 1.64MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5b139c2b3d714ab8a51f95184da81509"
          }
        },
        "749db83dd99c4c0b9607f9dc81e2a429": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "83e2e50b3af5464fa7078f6e6e0c158d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e9c2d26361644e1e847cfffd8b2f3cca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5b139c2b3d714ab8a51f95184da81509": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "883a40642f054744878161bea2965b00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_103c965a189742a596c303dcddcccc74",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_260d1650ec2d4d5986775c76519da671",
              "IPY_MODEL_9a2178890dc540719f9b10f8cfb96e4c"
            ]
          }
        },
        "103c965a189742a596c303dcddcccc74": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "260d1650ec2d4d5986775c76519da671": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_364d91569d90463eb74e0b1e772692f0",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_97408da268ca405ab8336cf4e82aa3cc"
          }
        },
        "9a2178890dc540719f9b10f8cfb96e4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d1e3c97773f746c297393d43cd5aaeaa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:24&lt;00:00, 17.3B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5d94cb7e8ff14bcd872f5b20ab6d717d"
          }
        },
        "364d91569d90463eb74e0b1e772692f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "97408da268ca405ab8336cf4e82aa3cc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d1e3c97773f746c297393d43cd5aaeaa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5d94cb7e8ff14bcd872f5b20ab6d717d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "857c899bb86b4c45bd3a74a264916d0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d0fb7ed378e744c0ac2792abcf27041c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5cec83c0d7ec4dedae6b42b34c06377e",
              "IPY_MODEL_c4c4f71844544061951325c133f60e4a"
            ]
          }
        },
        "d0fb7ed378e744c0ac2792abcf27041c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5cec83c0d7ec4dedae6b42b34c06377e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f54a205db6e149df8eacd6d37fe45a5d",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4fd90dca5338476f883a6b1c860980dc"
          }
        },
        "c4c4f71844544061951325c133f60e4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_def7a6cd52c043c58ee7a82aa6810806",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:06&lt;00:00, 70.4MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aa179a0723474285be7b51438bbfe29d"
          }
        },
        "f54a205db6e149df8eacd6d37fe45a5d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4fd90dca5338476f883a6b1c860980dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "def7a6cd52c043c58ee7a82aa6810806": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aa179a0723474285be7b51438bbfe29d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/KristianMiok/Bayesian-BERT/blob/main/MCD_BERT_EN.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JzsLKo5evIKr",
        "outputId": "cc1bcd86-a1c5-407d-da81-41bd216de6c0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "# Get the GPU device name.\n",
        "device_name = tf.test.gpu_device_name()\n",
        "\n",
        "# The device name should look like the following:\n",
        "if device_name == '/device:GPU:0':\n",
        "    print('Found GPU at: {}'.format(device_name))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T_h6VIL4vcFo",
        "outputId": "dd3e16ee-c64e-4136-b964-bb9466652acc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "import torch\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla P100-PCIE-16GB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WrTt0fQRvgqf",
        "outputId": "3cf354ad-a164-44b3-ccd7-a3d363e4801f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 663
        }
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a3/78/92cedda05552398352ed9784908b834ee32a0bd071a9b32de287327370b7/transformers-2.8.0-py3-none-any.whl (563kB)\n",
            "\r\u001b[K     |▋                               | 10kB 21.9MB/s eta 0:00:01\r\u001b[K     |█▏                              | 20kB 2.2MB/s eta 0:00:01\r\u001b[K     |█▊                              | 30kB 2.8MB/s eta 0:00:01\r\u001b[K     |██▎                             | 40kB 2.1MB/s eta 0:00:01\r\u001b[K     |███                             | 51kB 2.3MB/s eta 0:00:01\r\u001b[K     |███▌                            | 61kB 2.7MB/s eta 0:00:01\r\u001b[K     |████                            | 71kB 3.0MB/s eta 0:00:01\r\u001b[K     |████▋                           | 81kB 3.2MB/s eta 0:00:01\r\u001b[K     |█████▎                          | 92kB 3.6MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 102kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 112kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████                         | 122kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 133kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 143kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████▊                       | 153kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 163kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 174kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████▌                     | 184kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████                     | 194kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 204kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 215kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 225kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 235kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 245kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 256kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 266kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 276kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 286kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 296kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 307kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 317kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 327kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 337kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 348kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 358kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 368kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 378kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 389kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 399kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▎        | 409kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 419kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 430kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 440kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 450kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 460kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 471kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 481kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 491kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 501kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 512kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 522kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 532kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 542kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 552kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 563kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 573kB 3.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.12.47)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 15.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.3)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/98/2c/8df20f3ac6c22ac224fff307ebc102818206c53fc454ecd37d8ac2060df5/sentencepiece-0.1.86-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
            "\u001b[K     |████████████████████████████████| 1.0MB 20.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.38.0)\n",
            "Collecting tokenizers==0.5.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/3f/73c881ea4723e43c1e9acf317cf407fab3a278daab3a69c98dcac511c04f/tokenizers-0.5.2-cp36-cp36m-manylinux1_x86_64.whl (3.7MB)\n",
            "\u001b[K     |████████████████████████████████| 3.7MB 19.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: botocore<1.16.0,>=1.15.47 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (1.15.47)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.3.3)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.9.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.4.5.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.9)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.47->boto3->transformers) (2.8.1)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.47->boto3->transformers) (0.15.2)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=34c632a9fc46c3facb3680bf073137435a9f77edd59265dfe24e0ccfe2c2c9ed\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, sentencepiece, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.86 tokenizers-0.5.2 transformers-2.8.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "or5q9M2jvjy-",
        "outputId": "b542d0c8-c96d-45d5-d880-6f75b484ae90",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        }
      },
      "source": [
        "!pip install wget"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting wget\n",
            "  Downloading https://files.pythonhosted.org/packages/47/6a/62e288da7bcda82b935ff0c6cfe542970f04e29c756b0e147251b2fb251f/wget-3.2.zip\n",
            "Building wheels for collected packages: wget\n",
            "  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for wget: filename=wget-3.2-cp36-none-any.whl size=9682 sha256=5d4251ce9153df1c91ffef594febc804c230ea0304bae311670af3dbcf919f5d\n",
            "  Stored in directory: /root/.cache/pip/wheels/40/15/30/7d8f7cea2902b4db79e3fea550d7d7b85ecb27ef992b618f3f\n",
            "Successfully built wget\n",
            "Installing collected packages: wget\n",
            "Successfully installed wget-3.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m-ToZzHbvmr4",
        "outputId": "01da4dc5-7f0a-48ed-b53c-84587ed54bc0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import wget\n",
        "import os\n",
        "\n",
        "print('Downloading dataset...')\n",
        "\n",
        "# The URL for the dataset zip file.\n",
        "url = 'https://nyu-mll.github.io/CoLA/cola_public_1.1.zip'\n",
        "\n",
        "# Download the file (if we haven't already)\n",
        "if not os.path.exists('./cola_public_1.1.zip'):\n",
        "    wget.download(url, './cola_public_1.1.zip')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading dataset...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uw4kAmvVvomX",
        "outputId": "963590b0-5f89-44bb-bcda-0dce6fd05693",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "# Unzip the dataset (if we haven't already)\n",
        "if not os.path.exists('./cola_public/'):\n",
        "    !unzip cola_public_1.1.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  cola_public_1.1.zip\n",
            "   creating: cola_public/\n",
            "  inflating: cola_public/README      \n",
            "   creating: cola_public/tokenized/\n",
            "  inflating: cola_public/tokenized/in_domain_dev.tsv  \n",
            "  inflating: cola_public/tokenized/in_domain_train.tsv  \n",
            "  inflating: cola_public/tokenized/out_of_domain_dev.tsv  \n",
            "   creating: cola_public/raw/\n",
            "  inflating: cola_public/raw/in_domain_dev.tsv  \n",
            "  inflating: cola_public/raw/in_domain_train.tsv  \n",
            "  inflating: cola_public/raw/out_of_domain_dev.tsv  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5rS3jcOW9hGx"
      },
      "source": [
        "# Our data set!"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Sv9ofdu9jnt",
        "outputId": "481881be-d1ef-4a3c-dafd-4f5cd2be9252",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "# Access to resources\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive', force_remount=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vomc06VCEy1H"
      },
      "source": [
        "# Read data from file\n",
        "import pandas as pd\n",
        "from sklearn.utils import shuffle\n",
        "\n",
        "# Point to the file in Google Drive\n",
        "filename='/content/gdrive/My Drive/big_dataset.csv'\n",
        "#filename='/content/gdrive/My Drive/EN_HS/big_dataset.csv'\n",
        "df = pd.read_csv(filename, sep=',')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7xsr_L9mE-YQ",
        "outputId": "f0c4ee00-0c05-48e3-86e5-a0485c4996d3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "df.columns"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['id', 'label', 'tweet', 'normalized_text'], dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FMt7yZRDj8Hq",
        "outputId": "8de99484-d777-4b0e-f010-862dad7c5834",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "df = df.reset_index(drop=True)\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>label</th>\n",
              "      <th>tweet</th>\n",
              "      <th>normalized_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>5964</td>\n",
              "      <td>0</td>\n",
              "      <td>@user #nbaupdates  #nbadraft  #cleveland    f...</td>\n",
              "      <td>nbaupdates nbadraft cleveland father day pleas...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>15185</td>\n",
              "      <td>0</td>\n",
              "      <td>just saw tv commercial for @user milk. there's...</td>\n",
              "      <td>saw tv commercial milk nothing fair life dairy...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>23078</td>\n",
              "      <td>0</td>\n",
              "      <td>@user #gogirl #summer #camps #cardiff  #girls ...</td>\n",
              "      <td>gogirl summer camp cardiff girl confident book...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>7989</td>\n",
              "      <td>0</td>\n",
              "      <td>collection launch on the 25th june @user   #a ...</td>\n",
              "      <td>collection launch th june bedford aist bluesky</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>15333</td>\n",
              "      <td>0</td>\n",
              "      <td>exciting times ahead!! #greatplacetowork #next...</td>\n",
              "      <td>excite time ahead nextchapter roydsllp withking</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      id  ...                                    normalized_text\n",
              "0   5964  ...  nbaupdates nbadraft cleveland father day pleas...\n",
              "1  15185  ...  saw tv commercial milk nothing fair life dairy...\n",
              "2  23078  ...  gogirl summer camp cardiff girl confident book...\n",
              "3   7989  ...     collection launch th june bedford aist bluesky\n",
              "4  15333  ...    excite time ahead nextchapter roydsllp withking\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_w3To1lf5RDB",
        "outputId": "25d32400-f630-44f2-99a5-6cb756713f65",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "from sklearn.utils import shuffle\n",
        "df1= shuffle(df, random_state=500)\n",
        "df1.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>label</th>\n",
              "      <th>tweet</th>\n",
              "      <th>normalized_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2187</th>\n",
              "      <td>5214</td>\n",
              "      <td>0</td>\n",
              "      <td>love love love being surprised... but i love s...</td>\n",
              "      <td>love love love surprised love surprising peopl...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>116</th>\n",
              "      <td>3536</td>\n",
              "      <td>0</td>\n",
              "      <td>@user almost ready for the new addition to the...</td>\n",
              "      <td>almost ready new addition davis household dogf...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4973</th>\n",
              "      <td>31740</td>\n",
              "      <td>1</td>\n",
              "      <td>@user my video on the whole @user situation #b...</td>\n",
              "      <td>video whole situation boycottdelta expose trut...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>834</th>\n",
              "      <td>7388</td>\n",
              "      <td>0</td>\n",
              "      <td>south sudan unrest exacerbated by conflict amo...</td>\n",
              "      <td>south sudan unrest exacerbate conflict among c...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1370</th>\n",
              "      <td>14917</td>\n",
              "      <td>0</td>\n",
              "      <td>fans didn't follow me back</td>\n",
              "      <td>fan not follow back</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         id  ...                                    normalized_text\n",
              "2187   5214  ...  love love love surprised love surprising peopl...\n",
              "116    3536  ...  almost ready new addition davis household dogf...\n",
              "4973  31740  ...  video whole situation boycottdelta expose trut...\n",
              "834    7388  ...  south sudan unrest exacerbate conflict among c...\n",
              "1370  14917  ...                                fan not follow back\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xAhN2g8AlGlD",
        "outputId": "13e2430a-38af-4efc-b472-0229d45d52c1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "df2 = df1.reset_index(drop=True)\n",
        "df2.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>label</th>\n",
              "      <th>tweet</th>\n",
              "      <th>normalized_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>5214</td>\n",
              "      <td>0</td>\n",
              "      <td>love love love being surprised... but i love s...</td>\n",
              "      <td>love love love surprised love surprising peopl...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3536</td>\n",
              "      <td>0</td>\n",
              "      <td>@user almost ready for the new addition to the...</td>\n",
              "      <td>almost ready new addition davis household dogf...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>31740</td>\n",
              "      <td>1</td>\n",
              "      <td>@user my video on the whole @user situation #b...</td>\n",
              "      <td>video whole situation boycottdelta expose trut...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>7388</td>\n",
              "      <td>0</td>\n",
              "      <td>south sudan unrest exacerbated by conflict amo...</td>\n",
              "      <td>south sudan unrest exacerbate conflict among c...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>14917</td>\n",
              "      <td>0</td>\n",
              "      <td>fans didn't follow me back</td>\n",
              "      <td>fan not follow back</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      id  ...                                    normalized_text\n",
              "0   5214  ...  love love love surprised love surprising peopl...\n",
              "1   3536  ...  almost ready new addition davis household dogf...\n",
              "2  31740  ...  video whole situation boycottdelta expose trut...\n",
              "3   7388  ...  south sudan unrest exacerbate conflict among c...\n",
              "4  14917  ...                                fan not follow back\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nzDQv7aU5RDF"
      },
      "source": [
        "train = pd.DataFrame(df2.iloc[:4000,:])\n",
        "test = pd.DataFrame(df2.iloc[4000:,:])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RfMyYg325RDI"
      },
      "source": [
        "# Get the lists of sentences and their labels.\n",
        "sentences = train.tweet.values\n",
        "labels = train.label.values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HDlym6r9v0_n",
        "outputId": "0b15fa57-3e49-4aae-b70a-989b32b9b8b9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 83,
          "referenced_widgets": [
            "b693c13b37a64d7289775cc1b8544ab5",
            "eded4a106b3e47c2ad5e61d07f93aaeb",
            "6d0c09e222444dcbae1381a0ace60733",
            "16410aeb46d5454587056b100582e71c",
            "749db83dd99c4c0b9607f9dc81e2a429",
            "83e2e50b3af5464fa7078f6e6e0c158d",
            "e9c2d26361644e1e847cfffd8b2f3cca",
            "5b139c2b3d714ab8a51f95184da81509"
          ]
        }
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "\n",
        "# Load the BERT tokenizer.\n",
        "print('Loading BERT tokenizer...')\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading BERT tokenizer...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b693c13b37a64d7289775cc1b8544ab5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=231508, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6eNSbQmGv6tf",
        "outputId": "087e72ea-4d3b-45f7-8b66-e2ba6ee38257",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "# Print the original sentence.\n",
        "print(' Original: ', sentences[0])\n",
        "\n",
        "# Print the sentence split into tokens.\n",
        "print('Tokenized: ', tokenizer.tokenize(sentences[0]))\n",
        "\n",
        "# Print the sentence mapped to token ids.\n",
        "print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentences[0])))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Original:  love love love being surprised... but i love surprising people even more  \n",
            "Tokenized:  ['love', 'love', 'love', 'being', 'surprised', '.', '.', '.', 'but', 'i', 'love', 'surprising', 'people', 'even', 'more']\n",
            "Token IDs:  [2293, 2293, 2293, 2108, 4527, 1012, 1012, 1012, 2021, 1045, 2293, 11341, 2111, 2130, 2062]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oNeXWOGcwHGP",
        "outputId": "5404b846-17dd-47f0-9df0-a9c1fb2a7f82",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "input_ids = []\n",
        "\n",
        "# For every sentence...\n",
        "for sent in sentences:\n",
        "    # `encode` will:\n",
        "    #   (1) Tokenize the sentence.\n",
        "    #   (2) Prepend the `[CLS]` token to the start.\n",
        "    #   (3) Append the `[SEP]` token to the end.\n",
        "    #   (4) Map tokens to their IDs.\n",
        "    encoded_sent = tokenizer.encode(\n",
        "                        sent,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "\n",
        "                        # This function also supports truncation and conversion\n",
        "                        # to pytorch tensors, but we need to do padding, so we\n",
        "                        # can't use these features :( .\n",
        "                        #max_length = 128,          # Truncate all sentences.\n",
        "                        #return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                   )\n",
        "    \n",
        "    # Add the encoded sentence to the list.\n",
        "    input_ids.append(encoded_sent)\n",
        "\n",
        "# Print sentence 0, now as a list of IDs.\n",
        "print('Original: ', sentences[1])\n",
        "print('Token IDs:', input_ids[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original:  @user almost ready for the new addition to the davis household   #dogforlife \n",
            "Token IDs: [101, 1030, 5310, 2471, 3201, 2005, 1996, 2047, 2804, 2000, 1996, 4482, 4398, 1001, 3899, 29278, 15509, 102]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kDufI5aVwJuH",
        "outputId": "2d0d7348-46a9-4473-dcb8-477968754a4b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print('Max sentence length: ', max([len(sen) for sen in input_ids]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Max sentence length:  64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C9aO0sr2wNO4",
        "outputId": "eeb6bad4-a3e8-4a57-b764-2adbcbfed0a8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "# We'll borrow the `pad_sequences` utility function to do this.\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "# Set the maximum sequence length.\n",
        "# I've chosen 64 somewhat arbitrarily. It's slightly larger than the\n",
        "# maximum training sentence length of 47...\n",
        "MAX_LEN = 64\n",
        "\n",
        "print('\\nPadding/truncating all sentences to %d values...' % MAX_LEN)\n",
        "\n",
        "print('\\nPadding token: \"{:}\", ID: {:}'.format(tokenizer.pad_token, tokenizer.pad_token_id))\n",
        "\n",
        "# Pad our input tokens with value 0.\n",
        "# \"post\" indicates that we want to pad and truncate at the end of the sequence,\n",
        "# as opposed to the beginning.\n",
        "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", \n",
        "                          value=0, truncating=\"post\", padding=\"post\")\n",
        "\n",
        "print('\\nDone.')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Padding/truncating all sentences to 64 values...\n",
            "\n",
            "Padding token: \"[PAD]\", ID: 0\n",
            "\n",
            "Done.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "90_U5qV8wRYv"
      },
      "source": [
        "# Create attention masks\n",
        "attention_masks = []\n",
        "\n",
        "# For each sentence...\n",
        "for sent in input_ids:\n",
        "    \n",
        "    # Create the attention mask.\n",
        "    #   - If a token ID is 0, then it's padding, set the mask to 0.\n",
        "    #   - If a token ID is > 0, then it's a real token, set the mask to 1.\n",
        "    att_mask = [int(token_id > 0) for token_id in sent]\n",
        "    \n",
        "    # Store the attention mask for this sentence.\n",
        "    attention_masks.append(att_mask)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ucc-52MwUg_"
      },
      "source": [
        "# Use train_test_split to split our data into train and validation sets for\n",
        "# training\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Use 80% for training and 10% for validation.\n",
        "train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels, \n",
        "                                                            random_state=2018, test_size=0.1)\n",
        "# Do the same for the masks.\n",
        "train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,\n",
        "                                             random_state=2018, test_size=0.1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "29wced1HwXlg"
      },
      "source": [
        "# Convert all inputs and labels into torch tensors, the required datatype \n",
        "# for our model.\n",
        "train_inputs = torch.tensor(train_inputs)\n",
        "validation_inputs = torch.tensor(validation_inputs)\n",
        "\n",
        "train_labels = torch.tensor(train_labels)\n",
        "validation_labels = torch.tensor(validation_labels)\n",
        "\n",
        "train_masks = torch.tensor(train_masks)\n",
        "validation_masks = torch.tensor(validation_masks)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sKm3pv0UwbzA"
      },
      "source": [
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "# The DataLoader needs to know our batch size for training, so we specify it \n",
        "# here.\n",
        "# For fine-tuning BERT on a specific task, the authors recommend a batch size of\n",
        "# 16 or 32.\n",
        "\n",
        "# We set all of the tweets into the batches\n",
        "batch_size = 8\n",
        "\n",
        "# Create the DataLoader for our training set.\n",
        "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "# Create the DataLoader for our validation set.\n",
        "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n",
        "validation_sampler = SequentialSampler(validation_data)\n",
        "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XsugQQf3wikg",
        "outputId": "590ea5e7-3980-460b-e7bb-65aa3676a2b1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "train_inputs"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[  101,  2047,  2733,  ...,     0,     0,     0],\n",
              "        [  101,  3232,  2383,  ...,     0,     0,     0],\n",
              "        [  101,  1001,  6209,  ...,     0,     0,     0],\n",
              "        ...,\n",
              "        [  101,  1030,  5310,  ...,     0,     0,     0],\n",
              "        [  101,  6289, 23644,  ...,     0,     0,     0],\n",
              "        [  101,  1996, 13521,  ...,     0,     0,     0]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EHHRQfisUeV2",
        "outputId": "5f5f936e-45e0-473f-9aee-43d824bf477f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "883a40642f054744878161bea2965b00",
            "103c965a189742a596c303dcddcccc74",
            "260d1650ec2d4d5986775c76519da671",
            "9a2178890dc540719f9b10f8cfb96e4c",
            "364d91569d90463eb74e0b1e772692f0",
            "97408da268ca405ab8336cf4e82aa3cc",
            "d1e3c97773f746c297393d43cd5aaeaa",
            "5d94cb7e8ff14bcd872f5b20ab6d717d",
            "857c899bb86b4c45bd3a74a264916d0d",
            "d0fb7ed378e744c0ac2792abcf27041c",
            "5cec83c0d7ec4dedae6b42b34c06377e",
            "c4c4f71844544061951325c133f60e4a",
            "f54a205db6e149df8eacd6d37fe45a5d",
            "4fd90dca5338476f883a6b1c860980dc",
            "def7a6cd52c043c58ee7a82aa6810806",
            "aa179a0723474285be7b51438bbfe29d"
          ]
        }
      },
      "source": [
        "# Loading BERT that we need!\n",
        "\n",
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "\n",
        "# Load BertForSequenceClassification, the pretrained BERT model with a single \n",
        "# linear classification layer on top. \n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n",
        "    num_labels = 2, # The number of output labels--2 for binary classification.\n",
        "    #mnum_labels = 1, # The number of output labels--2 for binary classification.\n",
        "                    # You can increase this for multi-class tasks.   \n",
        "    output_attentions = False, # Whether the model returns attentions weights.\n",
        "    output_hidden_states = False, # Whether the model returns all hidden-states.\n",
        ")\n",
        "\n",
        "# Tell pytorch to run this model on the GPU.\n",
        "model.cuda()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "883a40642f054744878161bea2965b00",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=433, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "857c899bb86b4c45bd3a74a264916d0d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=440473133, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4gO_wjqWxOLI",
        "outputId": "d98882e3-ef43-4be1-ee7b-eaf4f4ea8b4b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 612
        }
      },
      "source": [
        "# Get all of the model's parameters as a list of tuples.\n",
        "params = list(model.named_parameters())\n",
        "\n",
        "print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n",
        "\n",
        "print('==== Embedding Layer ====\\n')\n",
        "\n",
        "for p in params[0:5]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== First Transformer ====\\n')\n",
        "\n",
        "for p in params[5:21]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== Output Layer ====\\n')\n",
        "\n",
        "for p in params[-4:]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The BERT model has 201 different named parameters.\n",
            "\n",
            "==== Embedding Layer ====\n",
            "\n",
            "bert.embeddings.word_embeddings.weight                  (30522, 768)\n",
            "bert.embeddings.position_embeddings.weight                (512, 768)\n",
            "bert.embeddings.token_type_embeddings.weight                (2, 768)\n",
            "bert.embeddings.LayerNorm.weight                              (768,)\n",
            "bert.embeddings.LayerNorm.bias                                (768,)\n",
            "\n",
            "==== First Transformer ====\n",
            "\n",
            "bert.encoder.layer.0.attention.self.query.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.query.bias                (768,)\n",
            "bert.encoder.layer.0.attention.self.key.weight            (768, 768)\n",
            "bert.encoder.layer.0.attention.self.key.bias                  (768,)\n",
            "bert.encoder.layer.0.attention.self.value.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.value.bias                (768,)\n",
            "bert.encoder.layer.0.attention.output.dense.weight        (768, 768)\n",
            "bert.encoder.layer.0.attention.output.dense.bias              (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.weight        (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.bias          (768,)\n",
            "bert.encoder.layer.0.intermediate.dense.weight           (3072, 768)\n",
            "bert.encoder.layer.0.intermediate.dense.bias                 (3072,)\n",
            "bert.encoder.layer.0.output.dense.weight                 (768, 3072)\n",
            "bert.encoder.layer.0.output.dense.bias                        (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.weight                  (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.bias                    (768,)\n",
            "\n",
            "==== Output Layer ====\n",
            "\n",
            "bert.pooler.dense.weight                                  (768, 768)\n",
            "bert.pooler.dense.bias                                        (768,)\n",
            "classifier.weight                                           (2, 768)\n",
            "classifier.bias                                                 (2,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VSPHYndSy6a2"
      },
      "source": [
        "optimizer = AdamW(model.parameters(),\n",
        "                  lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
        "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
        "                )\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i2g6BsrKy9TB"
      },
      "source": [
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "# Number of training epochs (authors recommend between 2 and 4)\n",
        "epochs = 4\n",
        "\n",
        "# Total number of training steps is number of batches * number of epochs.\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "# Create the learning rate scheduler.\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
        "                                            num_training_steps = total_steps)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mZrSSNOOzJsJ"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "# Function to calculate the accuracy of our predictions vs labels\n",
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Tna72gHzSaB"
      },
      "source": [
        "import time\n",
        "import datetime\n",
        "\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    # Round to the nearest second.\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    \n",
        "    # Format as hh:mm:ss\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f0gtycLSzZ4E",
        "outputId": "0af8a396-70bf-40f7-fd24-5eb533a146a4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# Fine tuning!\n",
        "\n",
        "import random\n",
        "\n",
        "# This training code is based on the `run_glue.py` script here:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "\n",
        "\n",
        "# Set the seed value all over the place to make this reproducible.\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "# Store the average loss after each epoch so we can plot them.\n",
        "loss_values = []\n",
        "\n",
        "\n",
        "# For each epoch...\n",
        "for epoch_i in range(0, epochs):\n",
        "    \n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "    \n",
        "    # Perform one full pass over the training set.\n",
        "\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    # Measure how long the training epoch takes.\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Reset the total loss for this epoch.\n",
        "    total_loss = 0\n",
        "\n",
        "    # Put the model into training mode. Don't be mislead--the call to \n",
        "    # `train` just changes the *mode*, it doesn't *perform* the training.\n",
        "    # `dropout` and `batchnorm` layers behave differently during training\n",
        "    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
        "    model.train()\n",
        "\n",
        "    # For each batch of training data...\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "\n",
        "        # Progress update every 40 batches.\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            # Calculate elapsed time in minutes.\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            \n",
        "            # Report progress.\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        # Unpack this training batch from our dataloader. \n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using the \n",
        "        # `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids \n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels \n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "        # Always clear any previously calculated gradients before performing a\n",
        "        # backward pass. PyTorch doesn't do this automatically because \n",
        "        # accumulating the gradients is \"convenient while training RNNs\". \n",
        "        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
        "        model.zero_grad()        \n",
        "\n",
        "        # Perform a forward pass (evaluate the model on this training batch).\n",
        "        # This will return the loss (rather than the model output) because we\n",
        "        # have provided the `labels`.\n",
        "        # The documentation for this `model` function is here: \n",
        "        # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "        outputs = model(b_input_ids, \n",
        "                    token_type_ids=None, \n",
        "                    attention_mask=b_input_mask, \n",
        "                    labels=b_labels)\n",
        "        \n",
        "        # The call to `model` always returns a tuple, so we need to pull the \n",
        "        # loss value out of the tuple.\n",
        "        loss = outputs[0]\n",
        "\n",
        "        # Accumulate the training loss over all of the batches so that we can\n",
        "        # calculate the average loss at the end. `loss` is a Tensor containing a\n",
        "        # single value; the `.item()` function just returns the Python value \n",
        "        # from the tensor.\n",
        "        total_loss += loss.item()\n",
        "\n",
        "        # Perform a backward pass to calculate the gradients.\n",
        "        loss.backward()\n",
        "\n",
        "        # Clip the norm of the gradients to 1.0.\n",
        "        # This is to help prevent the \"exploding gradients\" problem.\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        # Update parameters and take a step using the computed gradient.\n",
        "        # The optimizer dictates the \"update rule\"--how the parameters are\n",
        "        # modified based on their gradients, the learning rate, etc.\n",
        "        optimizer.step()\n",
        "\n",
        "        # Update the learning rate.\n",
        "        scheduler.step()\n",
        "\n",
        "    # Calculate the average loss over the training data.\n",
        "    avg_train_loss = total_loss / len(train_dataloader)            \n",
        "    \n",
        "    # Store the loss value for plotting the learning curve.\n",
        "    loss_values.append(avg_train_loss)\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(format_time(time.time() - t0)))\n",
        "        \n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "    # After the completion of each training epoch, measure our performance on\n",
        "    # our validation set.\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Put the model in evaluation mode--the dropout layers behave differently\n",
        "    # during evaluation.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables \n",
        "    eval_loss, eval_accuracy = 0, 0\n",
        "    nb_eval_steps, nb_eval_examples = 0, 0\n",
        "\n",
        "    # Evaluate data for one epoch\n",
        "    for batch in validation_dataloader:\n",
        "        \n",
        "        # Add batch to GPU\n",
        "        batch = tuple(t.to(device) for t in batch)\n",
        "        \n",
        "        # Unpack the inputs from our dataloader\n",
        "        b_input_ids, b_input_mask, b_labels = batch\n",
        "        \n",
        "        # Telling the model not to compute or store gradients, saving memory and\n",
        "        # speeding up validation\n",
        "        with torch.no_grad():        \n",
        "\n",
        "            # Forward pass, calculate logit predictions.\n",
        "            # This will return the logits rather than the loss because we have\n",
        "            # not provided labels.\n",
        "            # token_type_ids is the same as the \"segment ids\", which \n",
        "            # differentiates sentence 1 and 2 in 2-sentence tasks.\n",
        "            # The documentation for this `model` function is here: \n",
        "            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "            outputs = model(b_input_ids, \n",
        "                            token_type_ids=None, \n",
        "                            attention_mask=b_input_mask)\n",
        "        \n",
        "        # Get the \"logits\" output by the model. The \"logits\" are the output\n",
        "        # values prior to applying an activation function like the softmax.\n",
        "        logits = outputs[0]\n",
        "\n",
        "        # Move logits and labels to CPU\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "        \n",
        "        # Calculate the accuracy for this batch of test sentences.\n",
        "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
        "        \n",
        "        # Accumulate the total accuracy.\n",
        "        eval_accuracy += tmp_eval_accuracy\n",
        "\n",
        "        # Track the number of batches\n",
        "        nb_eval_steps += 1\n",
        "\n",
        "    # Report the final accuracy for this validation run.\n",
        "    print(\"  Accuracy: {0:.2f}\".format(eval_accuracy/nb_eval_steps))\n",
        "    print(\"  Validation took: {:}\".format(format_time(time.time() - t0)))\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 4 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/pytorch/torch/csrc/utils/python_arg_parser.cpp:756: UserWarning: This overload of add_ is deprecated:\n",
            "\tadd_(Number alpha, Tensor other)\n",
            "Consider using one of the following signatures instead:\n",
            "\tadd_(Tensor other, *, Number alpha)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "  Batch    40  of    450.    Elapsed: 0:00:04.\n",
            "  Batch    80  of    450.    Elapsed: 0:00:07.\n",
            "  Batch   120  of    450.    Elapsed: 0:00:10.\n",
            "  Batch   160  of    450.    Elapsed: 0:00:13.\n",
            "  Batch   200  of    450.    Elapsed: 0:00:16.\n",
            "  Batch   240  of    450.    Elapsed: 0:00:20.\n",
            "  Batch   280  of    450.    Elapsed: 0:00:23.\n",
            "  Batch   320  of    450.    Elapsed: 0:00:26.\n",
            "  Batch   360  of    450.    Elapsed: 0:00:29.\n",
            "  Batch   400  of    450.    Elapsed: 0:00:33.\n",
            "  Batch   440  of    450.    Elapsed: 0:00:36.\n",
            "\n",
            "  Average training loss: 0.38\n",
            "  Training epcoh took: 0:00:37\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.88\n",
            "  Validation took: 0:00:01\n",
            "\n",
            "======== Epoch 2 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    450.    Elapsed: 0:00:03.\n",
            "  Batch    80  of    450.    Elapsed: 0:00:06.\n",
            "  Batch   120  of    450.    Elapsed: 0:00:10.\n",
            "  Batch   160  of    450.    Elapsed: 0:00:13.\n",
            "  Batch   200  of    450.    Elapsed: 0:00:16.\n",
            "  Batch   240  of    450.    Elapsed: 0:00:19.\n",
            "  Batch   280  of    450.    Elapsed: 0:00:22.\n",
            "  Batch   320  of    450.    Elapsed: 0:00:25.\n",
            "  Batch   360  of    450.    Elapsed: 0:00:29.\n",
            "  Batch   400  of    450.    Elapsed: 0:00:32.\n",
            "  Batch   440  of    450.    Elapsed: 0:00:35.\n",
            "\n",
            "  Average training loss: 0.20\n",
            "  Training epcoh took: 0:00:36\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.89\n",
            "  Validation took: 0:00:01\n",
            "\n",
            "======== Epoch 3 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    450.    Elapsed: 0:00:03.\n",
            "  Batch    80  of    450.    Elapsed: 0:00:06.\n",
            "  Batch   120  of    450.    Elapsed: 0:00:09.\n",
            "  Batch   160  of    450.    Elapsed: 0:00:13.\n",
            "  Batch   200  of    450.    Elapsed: 0:00:16.\n",
            "  Batch   240  of    450.    Elapsed: 0:00:19.\n",
            "  Batch   280  of    450.    Elapsed: 0:00:22.\n",
            "  Batch   320  of    450.    Elapsed: 0:00:25.\n",
            "  Batch   360  of    450.    Elapsed: 0:00:28.\n",
            "  Batch   400  of    450.    Elapsed: 0:00:31.\n",
            "  Batch   440  of    450.    Elapsed: 0:00:35.\n",
            "\n",
            "  Average training loss: 0.09\n",
            "  Training epcoh took: 0:00:35\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.90\n",
            "  Validation took: 0:00:01\n",
            "\n",
            "======== Epoch 4 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    450.    Elapsed: 0:00:03.\n",
            "  Batch    80  of    450.    Elapsed: 0:00:06.\n",
            "  Batch   120  of    450.    Elapsed: 0:00:09.\n",
            "  Batch   160  of    450.    Elapsed: 0:00:13.\n",
            "  Batch   200  of    450.    Elapsed: 0:00:16.\n",
            "  Batch   240  of    450.    Elapsed: 0:00:19.\n",
            "  Batch   280  of    450.    Elapsed: 0:00:22.\n",
            "  Batch   320  of    450.    Elapsed: 0:00:25.\n",
            "  Batch   360  of    450.    Elapsed: 0:00:28.\n",
            "  Batch   400  of    450.    Elapsed: 0:00:31.\n",
            "  Batch   440  of    450.    Elapsed: 0:00:35.\n",
            "\n",
            "  Average training loss: 0.03\n",
            "  Training epcoh took: 0:00:35\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.90\n",
            "  Validation took: 0:00:01\n",
            "\n",
            "Training complete!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tpS2P37_1PXF",
        "outputId": "9c153b9d-60cd-4f85-871b-40d78b98231d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 461
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "% matplotlib inline\n",
        "\n",
        "import seaborn as sns\n",
        "\n",
        "# Use plot styling from seaborn.\n",
        "sns.set(style='darkgrid')\n",
        "\n",
        "# Increase the plot size and font size.\n",
        "sns.set(font_scale=1.5)\n",
        "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "# Plot the learning curve.\n",
        "plt.plot(loss_values, 'b-o')\n",
        "\n",
        "# Label the plot.\n",
        "plt.title(\"Training loss\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvAAAAGaCAYAAABpIXfbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd1xUZ9o+8GsGhqFXh14tgIJ0BBTFEhWxFxKNFcu6MfklMWtWXTVBd/OaWN6YbNZNNIolGFQCYo+KNVFpFkQBFZEiFgICgtKE3x95nV2CBRQ5Z+D6/jfPafd4f9CL43OeI6mvr68HERERERGpBKnQBRARERERUdMxwBMRERERqRAGeCIiIiIiFcIAT0RERESkQhjgiYiIiIhUCAM8EREREZEKYYAnImpn8vPz4eTkhH/+858vfY4FCxbAycmpBat6OU5OTliwYIHQZRARtSp1oQsgImrvmhOE4+PjYW1t/RqrISIisZPwRU5ERMKKi4tr8DklJQXbt2/HW2+9BW9v7wbbBg4cCG1t7Ve6Xn19Paqrq6GmpgZ19Ze7j1NTU4O6ujrI5fJXquVVOTk5YfTo0fj8888FrYOIqDXxDjwRkcBGjhzZ4PPjx4+xfft2eHh4NNr2R+Xl5dDV1W3W9SQSySsHb5lM9krHExHRy+MceCIiFdG/f39MnjwZV65cwYwZM+Dt7Y0RI0YA+D3If/nllwgNDYWfnx9cXV0xcOBArFq1Co8ePWpwnqfNgf/vsWPHjmHs2LHo3r07AgMD8cUXX6C2trbBOZ42B/7J2IMHD/Dpp58iICAA3bt3x/jx43Hx4sVG3+f+/ftYuHAh/Pz84OnpiSlTpuDKlSuYPHky+vfv/0p/Vjt37sTo0aPh5uYGb29vTJ8+HcnJyY32O378OCZNmgQ/Pz+4ubmhb9++eO+995Cdna3c5/bt21i4cCH69esHV1dXBAQEYPz48YiNjX2lGomIXhbvwBMRqZCCggJMnToVwcHBGDRoEB4+fAgAuHv3LqKjozFo0CAMGzYM6urqSExMxPfff4/09HRs2LChSec/ceIEtm3bhvHjx2Ps2LGIj4/Hxo0bYWBggD//+c9NOseMGTNgbGyMd999FyUlJYiIiMCf/vQnxMfHK/+3oLq6GmFhYUhPT8eYMWPQvXt3ZGZmIiwsDAYGBi/3h/N/Vq5cie+//x5ubm746KOPUF5ejh07dmDq1KlYu3YtgoKCAACJiYl455130KVLF8yePRt6enq4d+8ezpw5g9zcXDg4OKC2thZhYWG4e/cu3n77bdjb26O8vByZmZlITk7G6NGjX6lWIqKXwQBPRKRC8vPz8Y9//AOhoaENxm1sbHD8+PEGU1smTpyINWvW4N///jdSU1Ph5ub2wvNfv34de/fuVT4oO2HCBAwfPhw//PBDkwN8t27dEB4ervzcqVMnfPjhh9i7dy/Gjx8P4Pc75Onp6fjwww/xzjvvKPd1dHTEsmXLYGVl1aRr/dGNGzewYcMGeHl5YfPmzdDQ0AAAhIaGYujQoVi6dCkOHz4MNTU1xMfHo66uDhERETAxMVGe4913323w55GdnY158+Zh1qxZL1UTEVFL4xQaIiIVYmhoiDFjxjQa19DQUIb32tpalJaWori4GD179gSAp05heZoBAwY0WOVGIpHAz88PhYWFqKioaNI5pk2b1uCzv78/ACAnJ0c5duzYMaipqWHKlCkN9g0NDYWenl6TrvM08fHxqK+vx8yZM5XhHQDMzMwwZswY3Lp1C1euXAEA5XV+/vnnRlOEnniyT0JCAoqKil66LiKilsQ78EREKsTGxgZqampP3RYZGYmoqChcv34ddXV1DbaVlpY2+fx/ZGhoCAAoKSmBjo5Os89hZGSkPP6J/Px8mJqaNjqfhoYGrK2tUVZW1qR6/yg/Px8A0KVLl0bbnozl5eWhe/fumDhxIuLj47F06VKsWrUK3t7e6N27N4YNGwZjY2MAgJWVFf785z9j3bp1CAwMRNeuXeHv74/g4OAm/Y8GEdHrwDvwREQqREtL66njERERWLZsGUxNTbFs2TKsW7cOERERyuUVm7pi8LN+OWiJc4ht1WIjIyNER0djy5YtmDx5MioqKrB8+XIMHjwY58+fV+43d+5cHDp0CH/7299gY2OD6OhohIaGYuXKlQJWT0TtGe/AExG1AXFxcbCyssL69eshlf7n3szJkycFrOrZrKyscObMGVRUVDS4C19TU4P8/Hzo6+u/1Hmf3P2/du0abG1tG2y7fv16g32A33/Z8PPzg5+fHwAgIyMDY8eOxb///W+sW7euwXknT56MyZMno6qqCjNmzMD333+P6dOnN5g/T0TUGngHnoioDZBKpZBIJA3uctfW1mL9+vUCVvVs/fv3x+PHj7Fly5YG4zt27MCDBw9e6bwSiQQbNmxATU2NcvzevXuIiYmBlZUVunXrBgAoLi5udHzHjh0hl8uVU44ePHjQ4DwAIJfL0bFjRwBNn5pERNSSeAeeiKgNCA4OxurVqzFr1iwMHDgQ5eXl2Lt370u/afV1Cw0NRVRUFNasWYPc3FzlMpIHDx6EnZ3dMx8qfZGOHTsq745PmjQJQ4YMQUVFBXbs2IGHDx9i1apVyik+S5YswZ07dxAYGAhLS0tUVlbiwIEDqKioUL5AKyEhAUuWLMGgQYPg4OAAHR0dpKWlITo6Gu7u7sogT0TUmsT5NzsRETXLjBkzUF9fj+joaHz22WdQKBQYMmQIxo4di5CQEKHLa0RDQwObN2/GihUrEB8fjwMHDsDNzQ2bNm3CokWLUFlZ+dLn/vjjj2FnZ4dt27Zh9erVkMlkcHd3x+rVq+Hj46Pcb+TIkYiJiUFsbCyKi4uhq6uLzp074+uvv8bgwYMBAE5OThg4cCASExOxZ88e1NXVwcLCArNnz8b06dNf+c+BiOhlSOrF9lQRERG1W48fP4a/vz/c3Nya/PIpIqL2hnPgiYhIEE+7yx4VFYWysjL06tVLgIqIiFQDp9AQEZEgFi9ejOrqanh6ekJDQwPnz5/H3r17YWdnhzfffFPo8oiIRItTaIiISBC7du1CZGQkbt68iYcPH8LExARBQUH44IMP0KFDB6HLIyISLQZ4IiIiIiIVwjnwREREREQqhAGeiIiIiEiF8CHWZrp/vwJ1da0/68jERBdFReWtfl16NvZEnNgX8WFPxIl9ER/2RJyE6ItUKoGRkc4ztzPAN1NdXb0gAf7JtUlc2BNxYl/Ehz0RJ/ZFfNgTcRJbXziFhoiIiIhIhTDAExERERGpEAZ4IiIiIiIVwgBPRERERKRCGOCJiIiIiFQIAzwRERERkQphgCciIiIiUiEM8EREREREKoQBnoiIiIhIhfBNrCJ35vIdxJzIQnFZFYz15RgT1AkBLuZCl0VEREREAmGAF7Ezl+9g84EMVNfWAQCKyqqw+UAGADDEExEREbVTnEIjYjEnspTh/Ynq2jrEnMgSqCIiIiIiEhoDvIgVlVU1a5yIiIiI2j4GeBEz0Zc/dVxfW9bKlRARERGRWDDAi9iYoE7QUG/corKHNdhz+ibq6usFqIqIiIiIhMSHWEXsyYOq/70KzfBe9sjIKUHsyRu4lleCWcO7QU9bQ+BKiYiIiKi1MMCLXICLOQJczKFQ6KGw8AEAoLebJRxtDLHtyDWERyThnZGu6GxtIHClRERERNQaOIVGBUkkEvT1tMKiyd5QV5Pgi23ncDAhF/WcUkNERETU5jHAqzA7cz18Oq0HPDp3wI5j1/FNzCVUVNYIXRYRERERvUYM8CpOW1Mdc0a7YsKALkjNKsLSiCRk3y4TuiwiIiIiek0Y4NsAiUSCgb42WDDRC/X19Vj+QwriU/I5pYaIiIioDWKAb0M6WRng07Ae6GZvjMjDV/Ft3GU8qqoVuiwiIiIiakEM8G2MrpYM749zQ2jfTkjJLMSyTUnIvftA6LKIiIiIqIUwwLdBUokEQ/zt8PEED1TWPMZnW1Nw8mIBp9QQERERtQEM8G2Yk60Rlob1QBdrA2w6kIEN+9JRVf1Y6LKIiIiI6BUwwLdx+joa+OhND4wMdMCZtDv4+5ZkFPxWIXRZRERERPSSGODbAalUgpGBDvhovAcePKzGss1JOJN2R+iyiIiIiOglMMC3Iy72xggP6wF7c32s33sFmw9moKaWU2qIiIiIVAkDfDtjpCfHxxM8MDTADicuFOCzLSm4e/+h0GURERERUROpC3nx6upqfPXVV4iLi0NZWRmcnZ0xd+5cBAQEPPe43bt3Izo6GllZWSgtLYWpqSn8/Pzw3nvvwcrKqsG+Tk5OTz1HeHg4JkyY0GLfRZWoSaUYG9QJna0M8P3eK1gakYTpIV3h42wqdGlERERE9AKCBvgFCxbg0KFDmDJlCuzs7BAbG4tZs2Zh69at8PT0fOZxGRkZMDMzQ1BQEAwMDFBQUIAdO3bg+PHj2L17NxQKRYP9AwMDMWLEiAZj7u7ur+U7qRL3zh3waZgvvo27jLW70vCGtzXe7N8Z6mr8jxkiIiIisRIswKempmLfvn1YuHAhpk2bBgAYNWoUhg0bhlWrViEyMvKZx/71r39tNDZgwACMGTMGu3fvxowZMxps69ixI0aOHNmi9bcVHQy0sGCiF3Yey8Lh5DxkFZThnZEu6GCoJXRpRERERPQUgt1qPXjwIGQyGUJDQ5Vjcrkc48aNQ0pKCu7du9es81laWgIAysrKnrq9srISVVVVL19wG6auJsWEN7rg3dGuuFNcgaWbknDh2m9Cl0VERERETyFYgE9PT4eDgwN0dHQajLu5uaG+vh7p6ekvPEdJSQmKiopw6dIlLFy4EACeOn8+OjoaHh4ecHNzw/Dhw3H48OGW+RJtjLeTKT6d5gsTA018/VMqdh67jtrHdUKXRURERET/RbApNIWFhTAzM2s0/mT+elPuwA8ePBglJSUAAENDQ3zyySfw9/dvsI+npydCQkJgbW2N27dvY8uWLXjvvfewevVqDBs2rAW+SdtiaqSNRZO9ERV/HQcScnH9Vin+PNIVRnpyoUsjIiIiIggY4CsrKyGTyRqNy+W/B8WmTHf55ptv8PDhQ2RnZ2P37t2oqGj8htGoqKgGn0ePHo1hw4Zh5cqVGDp0KCQSSbPqNjHRbdb+LUmh0Gu1a300yQde3czxr50XsHRTEv4y0RteTlyl5o9asyfUdOyL+LAn4sS+iA97Ik5i64tgAV5TUxM1NTWNxp8E9ydB/nl8fX0BAEFBQRgwYACGDx8ObW1tTJo06ZnHaGtrY/z48Vi9ejVu3LiBTp06NavuoqJy1NXVN+uYlqBQ6KGw8EGrXtPFxgBLpvpgbWwawtedwfBe9hjRywFSafN+6WmrhOgJvRj7Ij7siTixL+LDnoiTEH2RSiXPvWks2Bx4hULx1GkyhYWFAABT0+bd7bWxsYGLiwv27Nnzwn0tLCwAAKWlpc26RntkYaKDxVN90LO7OXb/ehOrt19AaUW10GURERERtVuCBXhnZ2dkZ2c3mvZy8eJF5fbmqqysxIMHL/4NKS8vDwBgbGzc7Gu0R3KZGmYM7YawEGdk3SpF+MZEZObeF7osIiIionZJsAAfHByMmpoa7Ny5UzlWXV2NmJgYeHl5KR9wLSgoQFZWVoNji4uLG50vLS0NGRkZcHFxee5+9+/fx7Zt22BtbQ17e/sW+jbtQ283Syye4gMtuTpW/Hge+87cRF19608nIiIiImrPBJsD7+7ujuDgYKxatQqFhYWwtbVFbGwsCgoKsHz5cuV+8+fPR2JiIjIzM5Vj/fr1w5AhQ+Do6AhtbW1cv34dP/30E3R0dDBnzhzlfpGRkYiPj0ffvn1haWmJu3fvYvv27SguLsa//vWvVv2+bYW1qS6WTPXB5oMZ+OnEDVzNK8Ws4d2gq9X4gWQiIiIianmCBXgAWLFiBdasWYO4uDiUlpbCyckJ69atg7e393OPe/vtt3HmzBkcOXIElZWVUCgUCA4Oxpw5c2BjY6Pcz9PTE+fOncPOnTtRWloKbW1teHh4YPbs2S+8Bj2bllwds0e4wMnGED/GX0N4RCL+PNIVna0MhC6NiIiIqM2T1NdzDkRztKdVaJri5p0yrI1Nw/0HVQjt2wkDfW2avTSnqhJrT9o79kV82BNxYl/Ehz0RJ65CQ22Ovbk+wsN84dbJBFFHr+ObmEt4WNl4eVAiIiIiahkM8PTKtDVleG9Md4wf0AWpWUUIj0jCzTtlQpdFRERE1CYxwFOLkEgkGORrgwUTvVBXX4//2ZqCY+fywRlaRERERC2LAZ5aVCcrA4SH9UBXO2NsPXQV3+2+jEdVtUKXRURERNRmMMBTi9PVkuGDUDeMDeqIpIx7WLY5Gfn3yoUui4iIiKhNYICn10IqkWBogD3+OsETlVW1+PuWZJxKLRC6LCIiIiKVxwBPr5WTrRHCp/dAZysDROzPwIZ9V1BV81josoiIiIhUFgM8vXYGOhr4y1seGNHLHqcv3cE/tiTjdlGF0GURERERqSQGeGoVUqkEo3p3xEdveaCsohrLNiXj7OU7QpdFREREpHIY4KlVuTgYIzysB2zNdLFuzxVs+TkTNbWcUkNERETUVAzw1OqM9OT4eIInhvjZ4vj5W/hsawru3X8odFlEREREKoEBngShriZFaL/OeH+cG4pKK7F0UxJSMu8JXRYRERGR6DHAk6A8OnfAp2G+MDfWwb9i07DtyFXUPq4TuiwiIiIi0WKAJ8F1MNDCwkleeMPHGkeS8/F55DkUlVYKXRYRERGRKDHAkyioq0nx9huOmDPKFbeLKhAekYiL138TuiwiIiIi0WGAJ1HxcTbFJ9N8YaKvia+iUxF9PAuP6zilhoiIiOgJBngSHTMjbfxtsjeCPCyx/2wOVm47j/sPqoQui4iIiEgUGOBJlDRkapga7IxZw7sh5245wiMScflmsdBlEREREQmOAZ5ELcDFHEum+kBfWwP/G3UBcb9ko66uXuiyiIiIiATDAE+iZ9lBB4un+CDA1Rxxv2Tjf3dcQFlFtdBlEREREQmCAZ5UglxDDTOGdkXYEGdcyy/FpxGJyMy9L3RZRERERK2OAZ5UhkQiQW93Syye4gNNmRpW/ngB+8/moK6eU2qIiIio/WCAJ5VjY6qLT6b5wttJgejjWfg6OhXlj2qELouIiIioVTDAk0rSkqvjzyNdMGmQI67cLMbSiERkFZQKXRYRERHRa8cATypLIpGgv5c1Fk7yhkQiwec/nMPhpDzUc0oNERERtWEM8KTyHCz08WmYL9w6meDH+GtYG5uGh5W1QpdFRERE9FowwFOboKMpw3tjuuOt/p1x4fpvWLopETl3HghdFhEREVGLY4CnNkMikWBwD1vMf9sLtY/r8dnWFBw7f4tTaoiIiKhNYYCnNqeztQHCw3zhbGeIrT9nYv2eK6is5pQaIiIiahsEDfDV1dVYuXIlAgMD4ebmhjfffBNnzpx54XG7d+/GlClT0KtXL7i6uqJ///5YuHAhbt269dT9d+7ciSFDhqB79+4YPHgwIiMjW/qrkMjoaWvgw1B3jOnTEQnpd/H3zcnILywXuiwiIiKiVyZogF+wYAE2b96MESNGYNGiRZBKpZg1axbOnz//3OMyMjJgZmaG6dOnIzw8HKNGjcKpU6cwbtw4FBYWNtg3KioKixcvhqOjI5YsWQJ3d3csW7YMGzdufJ1fjURAKpFgWE97fDzeEw8ra/GPzcn49dJtocsiIiIieiWSeoEmCKempiI0NBQLFy7EtGnTAABVVVUYNmwYTE1Nm32X/PLlyxgzZgz++te/YsaMGQCAyspKBAUFwdvbG2vXrlXuO2/ePBw9ehQnTpyAnp5es65TVFSOurrW/yNTKPRQWMiHMl9WaXkVvtt9GRm5JQh0s8DEgY6Qy9Re6ZzsiTixL+LDnogT+yI+7Ik4CdEXqVQCExPdZ29vxVoaOHjwIGQyGUJDQ5Vjcrkc48aNQ0pKCu7du9es81laWgIAysrKlGMJCQkoKSnB22+/3WDfiRMnoqKiAidPnnyFb0CqxEBXjnnjPTG8pz1+Tb2Nz7Yk43ZRhdBlERERETWbYAE+PT0dDg4O0NHRaTDu5uaG+vp6pKenv/AcJSUlKCoqwqVLl7Bw4UIAQEBAgHL7lStXAACurq4NjnNxcYFUKlVup/ZBKpVgdJ+OmPumO0rKq7FsczISrtwVuiwiIiKiZlEX6sKFhYUwMzNrNK5QKACgSXfgBw8ejJKSEgCAoaEhPvnkE/j7+ze4hoaGBgwNDRsc92SsuXf5qW1w7WiC8DBffLv7Mr7bfRlX80owfkAXyNS5KBMRERGJn2ABvrKyEjKZrNG4XC4H8Pt8+Bf55ptv8PDhQ2RnZ2P37t2oqGg4JeJZ13hynaZc44+eNx/pdVMomjdfn55NodDDyvf7YOv+dMQcv468wnLMn+ILcxOdFx/8h/OQ+LAv4sOeiBP7Ij7siTiJrS+CBXhNTU3U1NQ0Gn8Sqp8E+efx9fUFAAQFBWHAgAEYPnw4tLW1MWnSJOU1qqurn3psVVVVk67xR3yItW0Z5m8LKxMtbNibjvdXH8eMoV3h5aho0rHsiTixL+LDnogT+yI+7Ik48SHW/6JQKJ46heXJMpCmpqbNOp+NjQ1cXFywZ8+eBteoqalRTrN5orq6GiUlJc2+BrVNnl0UCA/zhbmxFr6JuYSo+GuofVwndFlERERETyVYgHd2dkZ2dnajaS8XL15Ubm+uyspKPHjwn9+QunbtCgBIS0trsF9aWhrq6uqU24k6GGphwURvDPC2xqGkPHwReQ5FpZVCl0VERETUiGABPjg4GDU1Ndi5c6dyrLq6GjExMfDy8lI+4FpQUICsrKwGxxYXFzc6X1paGjIyMuDi4qIc8/f3h6GhIbZt29Zg3x9//BHa2tro06dPS34lUnEydSkmDnTEO6Ncceu3CoRHJCI16zehyyIiIiJqQLA58O7u7ggODsaqVatQWFgIW1tbxMbGoqCgAMuXL1fuN3/+fCQmJiIzM1M51q9fPwwZMgSOjo7Q1tbG9evX8dNPP0FHRwdz5sxR7qepqYn3338fy5YtwwcffIDAwEAkJydj9+7dmDdvHvT19Vv1O5Nq8HU2ha2pLtbuSsOanakYGmCHUb0doCblKjVEREQkPMECPACsWLECa9asQVxcHEpLS+Hk5IR169bB29v7uce9/fbbOHPmDI4cOYLKykooFAoEBwdjzpw5sLGxabDvxIkTIZPJsHHjRsTHx8PCwgKLFi3ClClTXudXIxVnZqyNRZO9se3INew7k4Pr+aWYPdIFhrrNf/CZiIiIqCVJ6uvrW39JFRXGVWjan9Npt7Hl50xoytQwe4QLutobA2BPxIp9ER/2RJzYF/FhT8SJq9AQqaCerhZYMtUXOloyrIq6gN2/ZAvySxwRERERwABP1CRWHXTwyVRf+LuYYdcv2fhyxwWUPGj+i8CIiIiIXhUDPFETyTXUMHNYN0wb4ozMvFJ88L/HcTWv5MUHEhEREbUgBniiZpBIJOjjbonFU7wh11DDim3ncSAhB3V8lISIiIhaCQM80UuwNdPDmrlB8HJSYOexLHzz0yWUP6oRuiwiIiJqBxjgiV6StqYM74x0wcSBjrh0owhLI5Jwo6BM6LKIiIiojWOAJ3oFEokEA7yt8bfJv7+7YPkPKTicnAeuzkpERESvCwM8UQtwsNBH+HRfdO9ogh+PXMPaXWl4WFkrdFlERETUBjHAE7UQHU0Z/t/Y7nizX2ecv/oblm1OQu5dvpCDiIiIWhYDPFELkkgkCPazxfyJnqiprcM/tqTgxIVbnFJDRERELYYBnug16GJtiE/DfOFka4jNBzPx/d4rqKzmlBoiIiJ6dQzwRK+JvrYG5r7pjtG9HXD2yl38fXMybhWWC10WERERqTgGeKLXSCqRYHgvB8x7ywMVlbX4+5Zk/HrpttBlERERkQpjgCdqBV3tjREe5gsHc31s2JeOiP3pqK55LHRZREREpIIY4IlaiaGuHPMmeGBYTzucSr2Nf2xJwZ3ih0KXRURERCqGAZ6oFalJpRjTpxM+DHVHSXkVlm1KQmL6XaHLIiIiIhXCAE8kALdOJggP84WVQgffxl1G5KGrqKmtE7osIiIiUgEM8EQCMdbXxPy3vTC4hw3iz+Vj+Q8pKCx5JHRZREREJHIM8EQCUleT4q3+XfD/xnTH3fuPsDQiCeevFgpdFhEREYkYAzyRCHg6KvBpmC8URlr4Z8wlbD96DbWPOaWGiIiIGmOAJxIJU0Mt/G2SN/p7WeHnxDys2HYexWWVQpdFREREIsMATyQiMnUpJg1ywp9HuiCvsBzhEUm4dKNI6LKIiIhIRBjgiUSoR1czfDrNF4a6Gliz4yJiTmbhcR2n1BAREREDPJFomRtrY/EUHwS6WWDv6RysjrqAkvIqocsiIiIigTHAE4mYhkwNYSFdMWNoV9woKEN4RBLSc+4LXRYREREJiAGeSAX06m6BxVN9oKOpjlVR57Hn9E3U1dcLXRYREREJgAGeSEVYK3SxZKoP/LqaIfbkDazZcREPHlYLXRYRERG1MgZ4IhWiqaGOWcO7YUqwEzJySxAekYRr+SVCl0VEREStiAGeSMVIJBL09bDCosnekKlJ8UXkeRxMyEU9p9QQERG1C+pCXry6uhpfffUV4uLiUFZWBmdnZ8ydOxcBAQHPPe7QoUPYv38/UlNTUVRUBAsLC/Tr1w9z5syBnp5eg32dnJyeeo7w8HBMmDChxb4LUWuzM9fDJ9N8EXEgHTuOXcfVvBLMGNYVOpoyoUsjIiKi10jQAL9gwQIcOnQIU6ZMgZ2dHWJjYzFr1ixs3boVnp6ezzxuyZIlMDU1xciRI2FpaYnMzExs3boVp06dwk8//QS5XN5g/8DAQIwYMaLBmLu7+2v5TkStSVtTHXNGueJISj52HL2OpRFJeGeUKxws9IUujYiIiF4TwQJ8amoq9u3bh4ULF2LatGkAgFGjRmHYsGFYtWoVIiMjn3ns119/DT8/vwZjrq6umD9/Pvbt24cxY8Y02NaxY0eMHDmyxb8DkRhIJBIM9LFBR0t9fLsrDct/SMFb/bugv5cVJBiJCNgAACAASURBVBKJ0OURERFRCxNsDvzBgwchk8kQGhqqHJPL5Rg3bhxSUlJw7969Zx77x/AOAG+88QYAICsr66nHVFZWoqqKL8GhtquTpQE+DesBF3tjRB6+im/jLuNRVa3QZREREVELEyzAp6enw8HBATo6Og3G3dzcUF9fj/T09Gad77fffgMAGBkZNdoWHR0NDw8PuLm5Yfjw4Th8+PDLF04kYrpaMvy/cW4I7dsJKZmFWLYpCbl3HwhdFhEREbUgwQJ8YWEhTE1NG40rFAoAeO4d+KdZv3491NTUMGjQoAbjnp6emDt3LtauXYtPPvkE1dXVeO+997B3796XL55IxKQSCYb42+Gvb3uiquYxPtuagpMXC7hKDRERURsh2Bz4yspKyGSNV8t48gBqc6a77NmzB9HR0Zg9ezZsbW0bbIuKimrwefTo0Rg2bBhWrlyJoUOHNnuOsImJbrP2b0kKhd6Ld6JWJeaeKBR6cOliitXbUrDpQAZy7pVjzlh3aMoFfXa9VYi5L+0VeyJO7Iv4sCfiJLa+CPYvuaamJmpqahqNPwnuf1xJ5lmSk5OxaNEi9O3bFx988MEL99fW1sb48eOxevVq3LhxA506dWpW3UVF5aira/07mQqFHgoLORVCTFSlJ++NcsXe0zcR90s2MnPu451RrrDqoPPiA1WUqvSlPWFPxIl9ER/2RJyE6ItUKnnuTWPBptAoFIqnTpMpLCwEgKdOr/mjjIwMvPPOO3BycsKXX34JNTW1Jl3bwsICAFBaWtqMiolUk1QqwYhAB/xlvAfKH1bj75uTcCbtjtBlERER0UsSLMA7OzsjOzsbFRUVDcYvXryo3P48ubm5mDlzJoyNjfHdd99BW1u7ydfOy8sDABgbGzezaiLV1c3eGJ+G9YC9uT7W772CTQcyUF3zWOiyiIiIqJkEC/DBwcGoqanBzp07lWPV1dWIiYmBl5cXzMzMAAAFBQWNloYsLCzE9OnTIZFIsGHDhmcG8eLi4kZj9+/fx7Zt22BtbQ17e/uW+0JEKsBIT46PJ3hgaIAdTl4swGdbU3C3+KHQZREREVEzCDYH3t3dHcHBwVi1ahUKCwtha2uL2NhYFBQUYPny5cr95s+fj8TERGRmZirHZs6ciby8PMycORMpKSlISUlRbrO1tVW+xTUyMhLx8fHo27cvLC0tcffuXWzfvh3FxcX417/+1XpflkhE1KRSjA3qhC7WBli/5wqWbkpCWEhX+Dq/eNoaERERCU/Q5ShWrFiBNWvWIC4uDqWlpXBycsK6devg7e393OMyMjIAAN9//32jbaNHj1YGeE9PT5w7dw47d+5EaWkptLW14eHhgdmzZ7/wGkRtnVunDggP64Fv49Lw711puOptjbf6d4a6mmD/MUdERERNIKnn4tDNwlVo6Im20pPax3WIPp6FQ0l5cLDQxzsjXdDBUEvosl5aW+lLW8KeiBP7Ij7siThxFRoiEh11NSnGD+iCd0d3x53ih1i6KQkXrv0mdFlERET0DAzwRAQA8HZS4NNpPuhgoIWvf0rFjmPXUfu4TuiyiIiI6A8Y4IlIydRIG3+b7IV+nlY4mJCLFT+eR3FZpdBlERER0X9hgCeiBmTqapg82AmzR7gg7145wiOSkJZdJHRZRERE9H8Y4Inoqfy6meGTqT4w0NXAl9svIvbkDUEe4CYiIqKGGOCJ6JksTHSweIoPenW3wJ7TN7F6+wWUVlQLXRYREVG7xgBPRM8ll6lh+tCuCAtxRtatUoRvTERGzn2hyyIiImq3GOCJqEl6u1li8RQfaMnVsTLqPPaevok6vkaCiIio1THAE1GTWZvqYslUH/ToaoaYkzewZudFPHjIKTVEREStiQGeiJpFS66OPw3vhimDnZCRcx/hEUm4fqtU6LKIiIjaDQZ4Imo2iUSCvp5WWDTZB+pqEnwReQ6HEnNRzyk1RERErx0DPBG9NDtzPXw6zRfunTsg6uh1fBNzCQ8ra4Qui4iIqE1jgCeiV6KtKcO7o10xfkAXpGYVITwiCTfvlAldFhERUZvFAE9Er0wikWCQrw0WTPRCXX09/mdrCo6ey+eUGiIioteAAZ6IWkwnKwOEh/VAN3tj/HDoKr7bfRmPqmqFLouIiKhNaZEAX1tbi59//hk7duxAYWFhS5ySiFSUrpYM749zw7i+nZCcUYhlm5ORd69c6LKIiIjaDPXmHrBixQokJCTgp59+AgDU19cjLCwMycnJqK+vh6GhIXbs2AFbW9sWL5aIVINUIkGIvx06Werj292X8Y8tyZg0yBG93SyFLo2IiEjlNfsO/KlTp+Dj46P8fPToUSQlJWHGjBlYvXo1AGDdunUtVyERqSwnWyOEh/VAZysDROzPwIa9V1BV/VjosoiIiFRas+/A37lzB3Z2dsrPx44dg7W1NebNmwcAuHbtGvbs2dNyFRKRSjPQ0cBf3vLA7l+zsefXm7h55wHeGeUKyw46QpdGRESkkpp9B76mpgbq6v/J/QkJCejZs6fys42NDefBE1EDUqkEo3p3xEdveaDsYTX+vjkZZy/fEbosIiIildTsAG9ubo7z588D+P1ue15eHnx9fZXbi4qKoK2t3XIVElGb4eJgjPCwHrAz08W6PVew5WAGamo5pYaIiKg5mj2FZujQoVi7di2Ki4tx7do16OrqIigoSLk9PT2dD7AS0TMZ6cnx8dueiD2Zjf1nc3DjdhnmjHKFqRF/8SciImqKZt+Bnz17NkaPHo0LFy5AIpHgiy++gL6+PgDgwYMHOHr0KAICAlq8UCJqO9SkUozr2wkfjHNDUWkllm5KQkrmPaHLIiIiUgmS+hZ8VWJdXR0qKiqgqakJmUzWUqcVlaKictTVtf7bJRUKPRQWPmj169KzsSct47fSR/j3rsvIvl2GN3ys8Wa/zlBXe/lXVLAv4sOeiBP7Ij7siTgJ0RepVAITE91nb2/Ji9XW1kJPT6/NhnciankdDLSwcJIXBvrY4EhyPpb/cA6/lT4SuiwiIiLRanaAP3HiBP75z382GIuMjISXlxc8PDzwl7/8BTU1NS1WIBG1fepqUkx4owvmjHLFneIKLI1IwoXrvwldFhERkSg1O8Bv2LABN27cUH7OysrC//zP/8DU1BQ9e/bE/v37ERkZ2aJFElH74ONsik+n+cLEQBNfR6di5/HreFxXJ3RZREREotLsAH/jxg24uroqP+/fvx9yuRzR0dH4/vvvERISgl27drVokUTUfpgaaWPRZG/09bTCgbO5WLntPO4/qBK6LCIiItFodoAvLS2FkZGR8vPp06fh7+8PXd3fJ9r36NED+fn5LVchEbU7MnU1TBnshD8N74acu+UIj0jE5exiocsiIiIShWYHeCMjIxQUFAAAysvLcenSJfj4+Ci319bW4vFjvpiFiF6dv4s5PpnmA31tDfzv9gvYdeqGIKtAERERiUmzX+Tk4eGBqKgodO7cGSdPnsTjx4/Rp08f5facnByYmpo26VzV1dX46quvEBcXh7KyMjg7O2Pu3LkvXEf+0KFD2L9/P1JTU1FUVAQLCwv069cPc+bMgZ6eXqP9d+7ciY0bNyI/Px+WlpaYMmUKJk6c2LwvTkSCsDDRweIpPvjhUCZ2/3oT1/JL8acRLjDQ0RC6NCIiIkE0+w78+++/j7q6Onz44YeIiYnBqFGj0LlzZwBAfX09jhw5Ai8vryada8GCBdi8eTNGjBiBRYsWQSqVYtasWTh//vxzj1uyZAmysrIwcuRILF68GIGBgdi6dSsmTJiAqqqGc2WjoqKwePFiODo6YsmSJXB3d8eyZcuwcePG5n51IhKIXEMNM4Z1Q1iIM67fKkV4RCIyc+8LXRYREZEgXupFTiUlJTh37hz09PTg6+urHC8tLcWuXbvg5+cHZ2fn554jNTUVoaGhWLhwIaZNmwYAqKqqwrBhw2BqavrclWwSEhLg5+fXYGzXrl2YP38+li9fjjFjxgAAKisrERQUBG9vb6xdu1a577x583D06FGcOHHiqXfsn4cvcqIn2BNh5N0rx9pdabh3/yHG9OmIIf52kEokyu3si/iwJ+LEvogPeyJObeZFToaGhujfv3+D8A4ABgYGmDp16gvDOwAcPHgQMpkMoaGhyjG5XI5x48YhJSUF9+49+7XqfwzvAPDGG28A+H1ZyycSEhJQUlKCt99+u8G+EydOREVFBU6ePPnCOolIXGxMdfHJVB/4OpvipxM38HV0Ksof8d0TRETUfjR7DvwTubm5iI+PR15eHgDAxsYGAwYMgK2tbZOOT09Ph4ODA3R0dBqMu7m5ob6+Hunp6U2eSw8Av/32+0tf/nuFnCtXrgBAg2UvAcDFxQVSqRRXrlzB0KFDm3wNIhIHLbk6Zo9wgaONIaLiryE8IhG9u1vgl0u3UVxWBWN9OcYEdUKAi7nQpRIREbW4lwrwa9aswfr16xutNrNy5UrMnj0bH3zwwQvPUVhYCDMzs0bjCoUCAJ57B/5p1q9fDzU1NQwaNKjBNTQ0NGBoaNhg3ydjzb0GEYmHRCJBfy9rdLTUx/9uv4C4X28qtxWVVWHzgQwAYIgnIqI2p9kBPjo6Gt9++y08PT0xc+ZMdOnSBQBw7do1bNiwAd9++y1sbGyU89CfpbKyEjKZrNG4XC4HgEYPoz7Pnj17EB0djdmzZzf4H4BnXePJdZpzjSeeNx/pdVMomjdfn14/9kR4CoUeNGPTUP6otsF4dW0ddv2SjRF9uwhUGf03/qyIE/siPuyJOImtL80O8Nu2bYO7uzu2bt0KdfX/HG5ra4ugoCBMnDgRP/zwwwsDvKamJmpqGs9bfRKqnwT5F0lOTsaiRYvQt2/fRnf+NTU1UV1d/dTjqqqqmnyN/8aHWOkJ9kQ8fiutfOp44f1HuH2nFOpqL/W4D7UQ/qyIE/siPuyJOLWJh1izsrIQEhLSILw/oa6ujpCQkAYPkj6LQqF46hSWwsJCAGjS/PeMjAy88847cHJywpdffgk1NbVG16ipqUFJSUmD8erqapSUlDRrjj0RiZeJ/rN/GV/43RkcSc5DVQ1fMEdERG1DswO8TCbDw4cPn7m9oqLimdNW/puzszOys7NRUVHRYPzixYvK7c+Tm5uLmTNnwtjYGN999x20tbUb7dO1a1cAQFpaWoPxtLQ01NXVKbcTkWobE9QJGuoN/zrTUJdicA8bGOtrYtuRa/h47Wns/jWbK9YQEZHKa3aA7969O7Zv365c9eW/FRUVYceOHXB3d3/heYKDg1FTU4OdO3cqx6qrqxETEwMvLy/lA64FBQWN7ugXFhZi+vTpkEgk2LBhA4yNjZ96DX9/fxgaGmLbtm0Nxn/88Udoa2s3eIMsEamuABdzTB3iDBN9OST4/Y781CHOeKt/Fyyc5I0FE73QyVIfu05l4+O1pxEVfw3FZU+fdkNERCR2zX6RU1JSEqZNmwYdHR2MHTtW+RbW69evIyYmBhUVFdi0aRN8fHxeeK4PPvgA8fHxmDp1KmxtbREbG4u0tDRs3rwZ3t7eAIDJkycjMTERmZmZyuNGjhyJjIwMzJw5E46Ojg3OaWtrC09PT+XnyMhILFu2DMHBwQgMDERycjJ27dqFefPmYdasWc356gA4B57+gz0Rp+f1Jf9eOQ4k5CDhyj1IJECAqzmG+NnCwkTnqftTy+DPijixL+LDnoiTGOfAv9SbWI8ePYq///3vuH37doNxS0tLfPLJJ+jbt2+TzlNVVYU1a9Zgz549KC0thZOTEz766CP07NlTuc/TAryTk9Mzzzl69Gh8/vnnDcZ27NiBjRs3Ij8/HxYWFpg8eTKmTJnSpBr/iAGenmBPxKkpffmt5BF+TszDydQC1NbWwctRgZAAOzhY6LdSle0Lf1bEiX0RH/ZEnNpMgAeAuro6pKWlIT8/H8DvL3JycXHBjh07sGXLFuzfv//lKhY5Bnh6gj0Rp+b0payiGkdS8nE0JR8Pq2rR1c4IIf526GZvBIlE8porbT/4syJO7Iv4sCfiJMYA/9JvYpVKpXBzc4Obm1uD8fv37yM7O/tlT0tE1Gr0dTQwpk9HDPGzxYkLBTiUlIvV2y/AzkwPIQF28HZUQCplkCciInF56QBPRNRWaMnVEexniwHe1jhz+Q4OJOTi37vSYGqkhSF+tujpagGZOteSJyIicWCAJyL6PzJ1Kfq4WyKwuwXOXS3EvrM52HwwE7t+ycYgXxv09bCClpx/bRIRkbD4LxER0R9IpRL4OJvC20mB9Jz72H82BzuPZWHv6Rz097LCQB8b6OtoCF0mERG1UwzwRETPIJFI0M3eGN3sjZF9uwwHzuZg/5kcHErKQ6CbBYJ72EJhqCV0mURE1M40KcBHREQ0+YTnzp176WKIiMTKwUIfc0Z3x53ihziYkIOTFwpw4nwBenQ1xRB/O9iYPnu1ACIiopbUpAD/xRdfNOukXH6NiNoqc2NtTBvSFSMDO+JwUh6OXbiFs1fuwq2TCUL87eBoYyh0iURE1MY1KcBv2bLldddBRKRSjPTkeLN/ZwztaYej527hcFIePo88h85WBgjxt4NbZxNIeTODiIhegyYF+B49erzuOoiIVJKOpgzDe9pjkK8Nfkm9jYMJufj6p1RYddDBEH9b9OhqBnU1LkFJREQthw+xEhG1ALlMDQO8rRHkYYmkjHvYfzYH3+9NR+zJGxjcwxa93S0hl6kJXSYREbUBDPBERC1IXU2KABdz+HczQ2pWEfafzcG2I9ew+9ebeMPHGv29rKGrJRO6TCIiUmEM8EREr4FEIoF75w5w79wBV/NKcOBsDnadysaBs7kI8rDEIF8bGOtrCl0mERGpIAZ4IqLXzNHGEI42hsi/V44DCTk4kpyP+JR8BLiaY4ifLSxMdIQukYiIVAgDPBFRK7E21cWs4S4Y3bsjfk7Mw8nUAvyaehtejgqEBNjBwUJf6BKJiEgFMMATEbWyDoZamDjIEcN72eNISj6OpuQj5WohutoZISTADt3sjPg+DSIieiYGeCIigejraGBMn44Y4meLExcKcCgpF6ujLsDOXA9D/e3g5aiAVMogT0REDTHAExEJTEuujmA/WwzwtsaZy3dw4GwO1u5Kg5mRFob42yHAxRwyda4lT0REv2OAJyISCZm6FH3cLRHY3QLnrhZi39kcbDqQgdhTNzDY1xZBHpbQkvOvbSKi9o7/EhARiYxUKoGPsym8nRS4knMf+8/kYMex69h7+ib6e1vhDW8b6OtoCF0mEREJhAGeiEikJBIJXOyN4WJvjOzbZdh/Ngf7Tufg58Q89HazwOAetlAYagldJhERtTIGeCIiFeBgoY93R3fH7aIKHEzIxYkLBTh+vgA9upkixM8O1qa6QpdIRESthAGeiEiFWJjoICykK0b17ohDSbk4fqEAZy/fhVsnE4T428HRxlDoEomI6DVjgCciUkFGenK81b8LhgbY49i5fBxOzsfnkefQ2doAIf52cOtkAinXkiciapMY4ImIVJiulgzDezlgUA9b/JJ6GwcTcvF1dCqsFDoI8bODb1dTqKtxCUoioraEAZ6IqA2Qy9QwwNsaQR6WSEq/h/1nc7B+7xXEnLyBYD9bBLpZQC5TE7pMIiJqAQzwRERtiLqaFAGu5vBzMUNqVhH2n8lB5OGriPslGwN9rNHf2xo6mjKhyyQiolfAAE9E1AZJJRJ4dO4Aj84dcDWvBPvP5iD2VDb2J+Sir4clBvnawkhPLnSZRET0EhjgiYjaOEcbQzjaGCLvXjkOJOTgcFI+jiTno6erOYL9bGFhoiN0iURE1AwM8ERE7YSNqS7+NNwFo3t3xM+JuTiVehu/pN6Gl5MCIf52cLDQF7pEIiJqAkEDfHV1Nb766ivExcWhrKwMzs7OmDt3LgICAp57XGpqKmJiYpCamoqrV6+ipqYGmZmZjfbLz8/HgAEDnnqO9evXo0+fPi3yPYiIVInCUAuTBjlhRC8HHEnJQ3zKLaRkFqKrnRFCAuzQzc4IEi5BSUQkWoIG+AULFuDQoUOYMmUK7OzsEBsbi1mzZmHr1q3w9PR85nEnTpzAzp074eTkBBsbG9y4ceO51xkxYgQCAwMbjDk7O7fIdyAiUlX6OhoY06cThvjZ4cSFAvyclIvVURdgZ66Hof528HJUQCplkCciEhvBAnxqair27duHhQsXYtq0aQCAUaNGYdiwYVi1ahUiIyOfeeyECRMwa9YsaGpq4rPPPnthgHdxccHIkSNbsnwiojZDS66OYD9bDPC2xpnLd3DgbA7W7kqDmZEWhvjbIcDFHDJ1riVPRCQWgv2NfPDgQchkMoSGhirH5HI5xo0bh5SUFNy7d++Zx3bo0AGamprNut7Dhw9RXV390vUSEbV1MnUp+rhb4rNZ/pgzyhWacnVsOpCBv357GgcTcvGoqlboEomICAIG+PT0dDg4OEBHp+HqB25ubqivr0d6enqLXeurr76Cp6cn3Nzc8NZbbyEpKanFzk1E1NZIpRL4OJvik6k++Mt4D1ia6GDHsev4eO1pxJzMQlkFb4YQEQlJsCk0hYWFMDMzazSuUCgA4Ll34JtKKpUiMDAQAwcOhKmpKXJycrBhwwaEhYVh06ZN8PHxeeVrEBG1VRKJBC72xnCxN0b27TLsP5uDfadz8HNiHnq7WSC4hy06GGoJXSYRUbsjWICvrKyETNb4bYBy+e8vFqmqqnrla1haWmLDhg0NxkJCQjB06FCsWrUKUVFRzT6niYnuK9f1shQKPcGuTU/HnogT+9LyFAo99HCzQv69B4g5dh3HUvJw/EIB+nhYYWz/LrB/wRKU7Ik4sS/iw56Ik9j6IliA19TURE1NTaPxJ8H9SZBvaWZmZhg6dCh27NiBR48eQUureXePiorKUVdX/1pqex6FQg+FhQ9a/br0bOyJOLEvr5dcAkzo3xnBvjY4lJSL4+cLcPxcPtw6mSDE3w6ONoaNjmFPxIl9ER/2RJyE6ItUKnnuTWPBArxCoXjqNJnCwkIAgKmp6Wu7toWFBerq6lBWVtbsAE9ERICRnhxv9e+CoQH2OHYuH4eT8/F55Dl0tjZAiL8d3DuZcC15IqLXRLCHWJ2dnZGdnY2KiooG4xcvXlRuf13y8vKgpqYGAwOD13YNIqL2QFdLhuG9HLByTk9MHOiI+2WV+Do6FZ9sTMSZy3fwuK5O6BKJiNocwQJ8cHAwampqsHPnTuVYdXU1YmJi4OXlpXzAtaCgAFlZWS91jeLi4kZjOTk52LdvH3x8fJq9FCURET2dXKaGAd7WWD47ALOGdQPqgfV7rmDhd2ex75cbqKp5LHSJRERthmBTaNzd3REcHIxVq1ahsLAQtra2iI2NRUFBAZYvX67cb/78+UhMTERmZqZy7NatW4iLiwMAXLp0CQCwdu1aAL/fue/fvz8AYOXKlcjLy4O/vz9MTU2Rm5urfHB1/vz5rfI9iYjaE3U1KQJczeHnYobUrCLsP5ODb2MvQU9bhjd8bNDfywo6mo0XMCAioqYTLMADwIoVK7BmzRrExcWhtLQUTk5OWLduHby9vZ97XH5+Pr766qsGY08+jx49Whnge/XqhaioKPzwww948OAB9PX10atXL7z33nvo0qXL6/lSREQEqUQCj84d4NG5A+49qMa2g+mIPXkD+8/moJ+HFQb62sBI7/UsVkBE1NZJ6uvrW39JFRXGVWjoCfZEnNgX8XnSk7x75ThwNgcJ6XehJpWgp6s5gv3sYG6sLXSJ7RJ/VsSHPREnrkJDRETtlo2pLv40wgWj+nTEz4m5+CX1Nk5dvA1vJwWG+NvB4QVryRMR0e8Y4ImIqFWZGmph8iAnjOjlgCPJeTh67haSMwvRzd4IIf526GpnxCUoiYiegwGeiIgEYaCjgbFBnRDib4fjF27hUGIeVkVdgL25HkL87eDlqIBUyiBPRPRHDPBERCQoLbk6hvjZ4Q1va5xOu4MDCblYuysNZsbaGOJniwAXc8jUBVv1mIhIdBjgiYhIFGTqagjysEJvN0ukXC3E/jM52HQgA7tO3cAgX1sEeVhCS85/toiI+DchERGJilQqga+zKXycFLhy8z72n83BjmPXsff0TfT3tsYbPtbQ19YQukwiIsEwwBMRkShJJBK4OBjDxcEYNwrKcOBsDvadvolDibno7WaJwT1s0MFQS+gyiYhaHQM8ERGJXkdLfbw7pjtuF1XgQEIujl+4hWPnb8GvmymG+NnB2vTZ6yUTEbU1DPBERKQyLEx0MD2kK0YFOuBQUh5OXCjAmct34d7JBCEBduhibSh0iURErx0DPBERqRxjfU2MH9AFw3ra4+i5fBxJzsfyH86hi7UBQvzt4NbJhGvJE1GbxQBPREQqS1dLhhG9HDDY1xanUgvwc2IuvopOhbVCB0P87dCjqynUpFyCkojaFgZ4IiJSeXINNbzhY4O+nlZITL+LA2dzsX7PFcSevIHBPWwR6GYBuUxN6DKJiFoEAzwREbUZ6mpS9HS1gL+LOVKvF2Hf2ZuIPHwVu3/Nxhs+NujvZQUdTZnQZRIRvRIGeCIianOkEgk8unSAe2cTXMsvxf6zOYg9eQP7z+agn4cVBvrawEhPLnSZREQvhQGeiIjaLIlEAkcbQzjaGCL37gMcTMjFz0m5OJKSh56u5gj2s4O5sbbQZRIRNQsDPBERtQu2Znr40wgXjOrTET8n5uKX1Ns4dfE2vJ0UCAmwg725vtAlEhE1CQM8ERG1K6aGWpg8yAkjejngSHIejp67heTMQnSzN0KIvx262hlxCUoiEjUGeCIiapcMdDQwNqgTQvztcPz/t3fnUVWd9/7H3+fAYRYQOICMIgo4ApKoGE2MJg2xNsY01pgoadLajPc29vYuY9P+bps2prcxg7XNukk0NdqkSbUojY1DjDY2jo0aiOKIKCAyiAEUZdCzf38gJxLAETjnyOe1Vtfqec5+3M/2cWd/2Hz3s784Q38+fAAAIABJREFUxtrtRcx9/wt6h/dg/IhYhiZYMZsV5EXE+SjAi4hIt+bt6c7dw2O5Iy2KzbtLWbWtkNdX7CYsyIe7h8eQPjAci7vWkhcR56EALyIiAljc3bgtJZLRQyLYcaCCf2w5wqJV+8j+rIBv3RzNrckReHvqsikijqf/EomIiFzEbDZxc1IoNyVayTvyFR9tPcoH6w+xcvMRxg6NYtxNUfj7eDh6mCLSjSnAi4iItMFkMjEwLoiBcUEcLqlh1dajrNx8hDXbCxmdHMFdw6IJCfB29DBFpBtSgBcREbmMPhH+PHXfYI5X1rJqayH/3HWMDTuPMXxAKHePiCXK6ufoIYpIN6IALyIicoV6Bfvy6Lf7c+/oONb+u4hPvyhhy54ykuODGZ8eS7+oQEcPUUS6AQV4ERGRqxTk78UD4/oxYWRv1u8oZt2OYl788076RQUwfkQsQ+KDtZa8iHQaBXgREZFr5Odt4Z5Rcdw1LIaNuSWs2V7IvGW5RFl9uXtELMP6h+Jm1hKUItKxFOBFRESuk6eHG3feFM3tqZFsyytj1bZC3vowj+UbD5MxPIZRg3vhYXFz9DBF5AahAC8iItJB3N3M3DK4F+mDwsk5dIKPthzlz2sPkP1ZAXfeFM3YoZH4eFkcPUwRcXEK8CIiIh3MbDKR2s9KSt8QDhRV8dHWQrI2HuajrUcZkxrJnTdF07OHp6OHKSIuyqEBvqGhgXnz5pGdnU1NTQ1JSUnMnDmT9PT0S/bLzc0lKyuL3NxcDhw4QGNjI/v3729zW5vNxsKFC/nLX/5CRUUFvXv35oknnmD8+PGdcUgiIiJ2JpOJxJieJMb0pLDsFKu3FbJmeyHrPi9i5KBe3D08hrAgH0cPU0RcjEOfrHn22Wd55513uOeee3juuecwm83MmDGDXbt2XbLfp59+ytKlSwGIjo6+5Lavvvoqc+fOZdSoUfziF78gIiKCmTNnsnr16g47DhERkcuJCevBj+4ZyIuPpTN6SASbd5fysze38vqK3RwprXH08ETEhZgMwzAcsePc3FwmT57M7Nmz+f73vw9AfX09EyZMIDQ0lHfffbfdvidOnMDPzw8vLy9eeOEFFi9e3OYd+LKyMsaNG8fUqVN57rnnADAMg2nTpnH8+HHWrVuH+SpXB6isPI3N1vV/ZVZrDyoqTnX5fqV9mhPnpHlxPpqTtlXXNrDu8yLW7yzmbP15BvbuyfgRsSTF9uySJSg1L85Hc+KcHDEvZrOJ4OD2XxDnsDvwq1evxmKxMHnyZHubp6cn999/Pzt27KC8vLzdviEhIXh5eV12H+vWraOxsZEHH3zQ3mYymZg6dSrHjh0jNzf3+g5CRETkGgX4evDd2+J56YlbmDwmnuKKWl56/wt+s/hzduwvx+aY+2si4gIcFuD37t1LXFwcvr6+LdqHDBmCYRjs3bu3Q/bh5+dHXFxcq30A5OXlXfc+REREroePlzt3j4jld0+kk5mRSO3Zc/xx+W5+/tY2/pVTwrnzNkcPUUScjMMeYq2oqCAsLKxVu9VqBbjkHfir2UdISEin7kNERKQjWNzdGJMSya1DIvh8fzkfbT3Kn1btY8VnBXzr5mhuTY7A21OLx4mIAwN8XV0dFkvrtXA9PZuW1aqvr++QfXh4eHToPi5Vj9TZrNYeDtu3tE1z4pw0L85Hc3J1vh3mz/jR8ew6UMHf1h/kg/WH+MeWo3x7VBzfGdWHAL+OWYJS8+J8NCfOydnmxWEB3svLi8bGxlbtzaG6OWRf7z4aGho6dB96iFWaaU6ck+bF+WhOrl10kDfP3D+E/JJqVm0t5IOPD7B8wyFGJ0dw17BoQgK8r/nP1rw4H82Jc3LGh1gdFuCtVmubJSwVFRUAhIaGdsg+Pv/8807dh4iISGeLjwjg6fsGU3KiltXbCvnnrmNs2HmM4QPCuHtEDFFWx/12WES6nsMeYk1KSqKgoIDa2toW7Tk5Ofbvr1f//v05ffo0BQUFbe6jf//+170PERGRrhIR4suj3+7P/z6ezh03RbHzQAX/b+F2fr8sl0PF1Y4enoh0EYcF+IyMDBobG+0vZIKmN7NmZWUxdOhQ+wOuJSUl5OfnX9M+xo0bh8Vi4b333rO3GYbB+++/T0REBMnJydd3ECIiIg4Q5O/FA+P68dKTI7l3VByHjlUz5887+O2fd5CbfwIHveJFRLqIw0pokpOTycjIYO7cuVRUVBATE8Py5cspKSnhxRdftG83a9Ystm/f3uJFTceOHSM7OxuAL7/8EoDXX38daLpzP3bsWADCw8PJzMzk7bffpr6+nsGDB7Nu3To+//xzXn311at+iZOIiIgz8fO2cM+oOO4aFsPG3BLWbC/ktaW5RFn9GD8ihpv7h+Kma53IDceh61H97ne/47XXXiM7O5vq6moSExN58803SUtLu2S/4uJi5s2b16Kt+fOkSZPsAR7gpz/9KQEBAXzwwQdkZWURFxfHyy+/zPjx4zv+gERERBzA08ONO2+K5vbUSLbllbFqWyFvfphH1sbDZAyPYdTgXnhY3Bw9TBHpICZDv2e7KlqFRpppTpyT5sX5aE66ns0wyDl0go+2HCW/pAZ/Hwt33BTN2KGR5ORXkvVpPidr6gny9+S+2+JJHxju6CELOleclVahERERkU5nNplI7WclpW8IB4qq+GhrIVkbD/P3TQXYDOw3oipr6nln1T4AhXgRF6LCOBERkRuUyWQiMaYnM7+XzC8fuRmzydTqt8gN52z87Z/XtliEiDiG7sCLiIh0AzFhPWg4Z2vzu5On6vn1O/+mX1Qg/aIC6BsVSIBv6zeZi4hzUIAXERHpJoL9PamsqW/V7u3phoe7Gxt2HWPtv4sACOvpbQ/0/aIDCevpjclk6uohi0gbFOBFRES6iftui+edVfta3In3cDcz7VuJpA8M59x5G0dLT3GwuJqDxVV8cegEn315HAB/H0uLQB8d6oe7mypxRRxBAV5ERKSbaH5Qtb1VaNzdzMRHBhAfGUDG8BhshkFp5RkOFlfZQ/2OAxUAeFjMxEcENAX6qED6RPjj7alYIdIVdKaJiIh0I+kDw0kfGH5FS+OZTSYiQnyJCPHltpRIAL46Vc+hY9UcLGoK9R9uPoJhgMkEMaE97Hfo+0UFEOjn2RWHJNLtKMCLiIjIFevZw5Obk0K5OSkUgLP15zhcUmO/S78xt4R1O4oBsAZ60S8qkIQLgT48yEd19CIdQAFeRERErpm3pzsD44IYGBcEwLnzNgrLTtsD/ZeHK9m8uxQAP2+LveSmX1QAseE9VEcvcg0U4EVERKTDuLuZ6RPhT58If+4aBoZhUPbVWXvJzcHiKnYdPAGAxd1Mn17+9ItuCvXxEQH4eCmaiFyOzhIRERHpNCaTifAgH8KDfBidHAFAdW0Dhy7coT9QVMVHWwqxGUcxmSDa6td0h/5CqO/ZQ3X0It+kAC8iIiJdKsDXg7TEUNISm+ro6xqa6+ib7tB/9uVxPtnZVEcfEuDVouymV4gvZtXRSzenAC8iIiIO5eXhzoDeQQzo3VRHf95mo6j8NAeLmgL9niNfsWVPGQC+Xu70jfx6pZve4f5Y3FVHL92LAryIiIg4FTezmd7h/vQO9+fOm6MxDIOKqrP2O/QHi6vJya8Emmru43r1uLDaTdMa9r5eFgcfgUjnUoAXERERp2YymQjt6UNoTx9uGdwLgJozDRy6KNCv2V7IR1sNTECk1ffrt8ZGBRIc4OXYAxDpYArwIiIi4nL8fTwYmmBlaIIVgPrG8xRctB79lj2lbNh1DIAgf88WgT7Sqjp6cW0K8CIiIuLyPC1uJMX2JCm2JwA2m0FxxWl72c3+wq/YltdUR+/t6X4hzDcF+rhePbC4uzly+CJXRQFeREREbjhms4mYsB7EhPVgXFoUhmFQWV3XtHTlhbv0ufY6ehO9w/3tgb5vVAB+3qqjF+elAC8iIiI3PJPJREigNyGB3qQPCgfg9NnGFnX0a/9dxKpthQBEhPi2uEsfEuCFSWU34iQU4EVERKRb8vO2kNIvhJR+IQA0NJ7nSOkpe6DfvrecT78oASDQz+PCSjdNtfRRVj/MZgV6cQwFeBERERHAw+JGQnRTSAewGQbHKmrtgf5AURX/3lcOgJeHW9N69M119BH+eFpURy9dQwFeREREpA1mk4noUD+iQ/0YOzQK4EIdfZX94dgV/yrAANzMJmLDe7Soo/f38XDsAcgNSwFeRERE5AoFB3gRHBDOiIFNdfS1dY3kH6tuCvRFVXyy4xhrthcB0CvYxx7o+0UFYA30Vh29dAgFeBEREZFr5OtlYUh8CEPim+roG8/ZOFp6qmmlm6IqduyvYGPOcQACfD2+DvTRAUSH+uFmNjty+OKiFOBFREREOojF3UzfqAD6RgXAiFhshsHxE7X2kpuDxdV8vr8CaFq7Pj7S336Hfpi/t4NHL65CAV5ERESkk5hNJiKtfkRa/RiTGgnAyZo6Dh2r5mBRU6j/+2dNdfTmv+YQE+pnX+mmb1QgAb6qo5fWFOBFREREulCQvxfD/L0Y1j8MgDN15zhcUk3xybPk7C9nw65jrP13Ux19WE9v+x36ftGBhPVUHb0owIuIiIg4lI+XO4P6BHP78B5k3BTFufNNdfTNZTdfHDrBZ1821dH38LF8HeijAokJ88PdTXX03Y0CvIiIiIgTcXczEx8ZQHxkABnDYzAMg9KTZ+wr3RwsrmbngaY6eg+LmfiIr9ej7xPhj7en4t2NTjMsIiIi4sRMJhO9gn3pFezLrckRAFSdrufQhZdLHSyu5sPNRzAMMJkgJrSHveSmX1QAgX6eDj4C6WgODfANDQ3MmzeP7OxsampqSEpKYubMmaSnp1+2b1lZGXPmzGHTpk3YbDZGjBjB7NmziY6ObrFdYmJim/1/+ctfMnXq1A45DhEREZGuFOjnyU1JodyUFArA2fpzHC6psa90szG3hHU7igGwBnq1KLvpFeyjOnoX59AA/+yzz7J27VoyMzOJjY1l+fLlzJgxgyVLlpCamtpuv9raWjIzM6mtreXxxx/H3d2dRYsWkZmZyYoVKwgICGix/ahRo7jnnntatCUnJ3fKMYmIiIh0NW9PdwbGBTEwLgiAc+dtFJWftpfc7D5cyebdpQD4eVtavGAqNryH6uhdjMMCfG5uLv/4xz+YPXs23//+9wG49957mTBhAnPnzuXdd99tt+97773H0aNHycrKYsCAAQCMHj2a73znOyxatIgf//jHLbbv06cPEydO7LRjEREREXEm7m5m4nr5E9fLn28NA8MwKP/qbNMLpi7U0u86eAJoWru+Ty9/+kU3hfr4iAB8vFRl7cwcNjurV6/GYrEwefJke5unpyf3338/r776KuXl5YSGhrbZd82aNaSkpNjDO0B8fDzp6emsWrWqVYAHqKurw2Qy4empOjARERHpXkwmE2FBPoQF+TB6SFMdfXVtA4eaA31xFR9tKcRmHMUERIX6tbhLH+Tv5dgDkBYcFuD37t1LXFwcvr6+LdqHDBmCYRjs3bu3zQBvs9nYv38/U6ZMafXd4MGD2bRpE2fPnsXb++u3mS1btowlS5ZgGAYJCQn853/+J3feeWfHH5SIiIiIiwjw9SAtMZS0xKa8Vd9wnsMl1fZAv2l3Ket3HgMgJMCrRaDvFeKLWXX0DuOwAF9RUUFYWFirdqvVCkB5eXmb/aqqqmhoaLBv982+hmFQUVFBTEwMAKmpqYwfP56oqCiOHz/O4sWLefrpp3n55ZeZMGFCBx6RiIiIiOvy9HCjf+8g+vduqqM/b7NRXF7bVHZTVEXeka/YsqcMAF8vd/pGfr3STe9wfyzuqqPvKg4L8HV1dVgsllbtzSUu9fX1bfZrbvfwaP1q4ea+dXV19rb333+/xTaTJk1iwoQJvPTSS3z729++6qewg4P9rmr7jmS19nDYvqVtmhPnpHlxPpoT56R5cT7ONifhYQHcNLip5MYwDEorz5BXUElewUnyCirJ+Wc+0FRH3y86kAFxwQyIa/ohwM+ndVZzVc42Lw4L8F5eXjQ2NrZqbw7o7dWqN7c3NDS029fLq/06LR8fHx544AFefvllDh8+THx8/FWNu7LyNDabcVV9OoLV2oOKilNdvl9pn+bEOWlenI/mxDlpXpyPK8yJOzCkd0+G9O4Jt8dTc6aB/OKvy26W//MQy9Y35aRIqy8JFy1fGRzgmnX0jpgXs9l0yZvGDgvwVqu1zTKZioqmN4u19wBrYGAgHh4e9u2+2ddkMrVZXnOxXr16AVBdXX21wxYRERGRC/x9PEhNsJKa0JS96hvPc+R4DQcuBPqteaVs2NVURx/k79liPfrIEF/MZtXRXwuHBfikpCSWLFlCbW1tiwdZc3Jy7N+3xWw2k5CQwO7du1t9l5ubS2xsbIsHWNtSVFQEQFBQ0LUOX0RERES+wdPiRmJMTxJjegJgsxkUV5y236HfX/gV2/Ka6ui9PS/U0Uc1/a9PhD8WdzdHDt9lOCzAZ2Rk8Pbbb7N06VL7OvANDQ1kZWUxdOhQ+wOuJSUlnD17tkWpy1133cUrr7xCXl6efSnJw4cPs3XrVmbMmGHf7uTJk61C+ldffcV7771HVFQUvXv37tyDFBEREenGzGYTMWE9iAnrwbi0KAzDoLK6zh7oDxZXk7WxEgB3NxO9w/3td+j7RgXg5936eUlxYIBPTk4mIyODuXPn2leNWb58OSUlJbz44ov27WbNmsX27dvZv3+/ve3BBx9k6dKl/OhHP+KRRx7Bzc2NRYsWYbVa7T8MALz77rt88sknjBkzhoiICMrKyvjggw84efIkf/zjH7vycEVERES6PZPJREigNyGB3qQPCgfg9NlGDh37OtB//HkRq7YVAhAR4mu/Q98vKpCQAK+rXoDkRuTQ12z97ne/47XXXiM7O5vq6moSExN58803SUtLu2Q/Pz8/lixZwpw5c3j99dex2WwMHz6c5557jp49e9q3S01NZefOnSxdupTq6mp8fHxISUnhscceu+w+RERERKTz+XlbSOkbQkrfEAAaz52n4Pgpe6DfvrecT78oASDQz6NFHX10qF+3rKM3GYbR9UuquDCtQiPNNCfOSfPifDQnzknz4nw0J22zGQYlFbX2QH+wuIrKmgsrD3q4XVRHH0hchD+elo6to9cqNCIiIiIiV8FsMhEV6kdUqB+3D40CaKqjP3Yh0BdVseJfBRiAm9lEbHiPFnX0/jfQevTNFOBFRERExKUEB3gRHBDOiAFNdfS1dY3kH6u2B/pPdhxjzfamVQfDg3zsgT4hOgBroLfL19ErwIuIiIiIS/P1sjAkPoQh8c119DaOln5dR7/zQAX/yj0OQICvhz3Q94sOIDrUDzezudWfuWVPKVmf5nOypp4gf0/uuy2e9IHhXXpc7VGAFxEREZEbisXdTN+oAPpGBXA3TXX0xyvPNAX6oqY6+s/3N70U1NPiRnykv/3h2D4R/uw6eIJ3Vu2j4ZwNgMqaet5ZtQ/AKUK8AryIiIiI3NDMJhORIb5EhvgyJiUSgJM1dU3LV14I9H//rKmO3mwyYTLB+W8sWtJwzkbWp/kK8CIiIiIijhDk78Uwfy+G9W96eeiZunMcLqnmQHE1KzcfabNP8+o3jta64EdEREREpJvx8XJnUJ9g7ru1D8H+nm1u0157V1OAFxERERG5yH23xePh3jIme7ibue+2eAeNqCWV0IiIiIiIXKS5zl2r0IiIiIiIuIj0geGkDwx3yjfkqoRGRERERMSFKMCLiIiIiLgQBXgREREREReiAC8iIiIi4kIU4EVEREREXIgCvIiIiIiIC1GAFxERERFxIQrwIiIiIiIuRAFeRERERMSF6E2sV8lsNnXLfUvbNCfOSfPifDQnzknz4nw0J86pq+flcvszGYZhdNFYRERERETkOqmERkRERETEhSjAi4iIiIi4EAV4EREREREXogAvIiIiIuJCFOBFRERERFyIAryIiIiIiAtRgBcRERERcSEK8CIiIiIiLkQBXkRERETEhSjAi4iIiIi4EHdHD6A7a2hoYN68eWRnZ1NTU0NSUhIzZ84kPT39sn3LysqYM2cOmzZtwmazMWLECGbPnk10dHQXjPzGda1zMn/+fP7whz+0ag8JCWHTpk2dNdxuoby8nMWLF5OTk8Pu3bs5c+YMixcvZvjw4VfUPz8/nzlz5rBz504sFgu33347s2bNIigoqJNHfmO7nnl59tlnWb58eav25ORk/vrXv3bGcLuF3Nxcli9fzrZt2ygpKSEwMJDU1FSeeeYZYmNjL9tf15WOdz1zoutK5/nyyy/5v//7P/Ly8qisrKRHjx4kJSXx1FNPMXTo0Mv2d4ZzRQHegZ599lnWrl1LZmYmsbGxLF++nBkzZrBkyRJSU1Pb7VdbW0tmZia1tbU8/vjjuLu7s2jRIjIzM1mxYgUBAQFdeBQ3lmudk2bPP/88Xl5e9s8X/3+5NgUFBbz11lvExsaSmJjIrl27rrhvaWkpDz30EP7+/sycOZMzZ87w9ttvc+DAAf76179isVg6ceQ3tuuZFwBvb29+9atftWjTD1XXZ8GCBezcuZOMjAwSExOpqKjg3Xff5d5772XZsmXEx8e321fXlc5xPXPSTNeVjldUVMT58+eZPHkyVquVU6dO8eGHHzJt2jTeeustbrnllnb7Os25YohD5OTkGAkJCcaf/vQne1tdXZ1xxx13GA8++OAl+7755ptGYmKisWfPHnvboUOHjP79+xuvvfZaZw35hnc9c/L73//eSEhIMKqrqzt5lN3PqVOnjJMnTxqGYRgff/yxkZCQYGzduvWK+v7P//yPkZKSYpSWltrbNm3aZCQkJBhLly7tlPF2F9czL7NmzTLS0tI6c3jd0o4dO4z6+voWbQUFBcagQYOMWbNmXbKvriud43rmRNeVrnXmzBlj5MiRxo9+9KNLbucs54pq4B1k9erVWCwWJk+ebG/z9PTk/vvvZ8eOHZSXl7fbd82aNaSkpDBgwAB7W3x8POnp6axatapTx30ju545aWYYBqdPn8YwjM4carfi5+dHz549r6nv2rVrGTt2LGFhYfa2kSNH0rt3b50r1+l65qXZ+fPnOX36dAeNSIYOHYqHh0eLtt69e9OvXz/y8/Mv2VfXlc5xPXPSTNeVruHt7U1QUBA1NTWX3M5ZzhUFeAfZu3cvcXFx+Pr6tmgfMmQIhmGwd+/eNvvZbDb279/PoEGDWn03ePBgjhw5wtmzZztlzDe6a52Ti40ZM4a0tDTS0tKYPXs2VVVVnTVcuYyysjIqKyvbPFeGDBlyRfMpnae2ttZ+rgwfPpwXX3yR+vp6Rw/rhmMYBidOnLjkD1u6rnStK5mTi+m60nlOnz7NyZMnOXz4MK+88goHDhy45DNvznSuqAbeQSoqKlrcFWxmtVoB2r3bW1VVRUNDg327b/Y1DIOKigpiYmI6dsDdwLXOCYC/vz/Tp08nOTkZi8XC1q1b+eCDD8jLy2Pp0qWt7sBI52uer/bOlcrKSs6fP4+bm1tXD63bs1qt/PCHP6R///7YbDY2bNjAokWLyM/PZ8GCBY4e3g3l73//O2VlZcycObPdbXRd6VpXMieg60pX+NnPfsaaNWsAsFgsPPDAAzz++OPtbu9M54oCvIPU1dW1+QCdp6cnQLt3oprb2zpxm/vW1dV11DC7lWudE4CHH364xeeMjAz69evH888/z4oVK/je977XsYOVy7rSc+Wbv3GRzvdf//VfLT5PmDCBsLAwFi5cyKZNmy75AJlcufz8fJ5//nnS0tKYOHFiu9vputJ1rnROQNeVrvDUU08xZcoUSktLyc7OpqGhgcbGxnZ/OHKmc0UlNA7i5eVFY2Njq/bmfxzN/xC+qbm9oaGh3b56Qv3aXOuctGfq1Kl4e3uzZcuWDhmfXB2dK67l0UcfBdD50kEqKip47LHHCAgIYN68eZjN7V/uda50jauZk/boutKxEhMTueWWW/jud7/LwoUL2bNnD7Nnz253e2c6VxTgHcRqtbZZklFRUQFAaGhom/0CAwPx8PCwb/fNviaTqc1f7cjlXeuctMdsNhMWFkZ1dXWHjE+uTvN8tXeuBAcHq3zGiYSEhGCxWHS+dIBTp04xY8YMTp06xYIFCy57TdB1pfNd7Zy0R9eVzmOxWBg3bhxr165t9y66M50rCvAOkpSUREFBAbW1tS3ac3Jy7N+3xWw2k5CQwO7du1t9l5ubS2xsLN7e3h0/4G7gWuekPY2NjRw/fvy6V+qQaxMWFkZQUFC750r//v0dMCppT2lpKY2NjVoL/jrV19fz+OOPc+TIEd544w369Olz2T66rnSua5mT9ui60rnq6uowDKNVDmjmTOeKAryDZGRk0NjYyNKlS+1tDQ0NZGVlMXToUPvDlCUlJa2Wmrrrrrv44osvyMvLs7cdPnyYrVu3kpGR0TUHcAO6njk5efJkqz9v4cKF1NfXM3r06M4duABQWFhIYWFhi7ZvfetbrF+/nrKyMnvbli1bOHLkiM6VLvLNeamvr29z6cjXX38dgFGjRnXZ2G4058+f55lnnuGLL75g3rx5pKSktLmdritd53rmRNeVztPW3+3p06dZs2YNvXr1Ijg4GHDuc8VkaGFRh/nxj3/MJ598wsMPP0xMTAzLly9n9+7dvPPOO6SlpQEwffp0tm/fzv79++39Tp8+zaRJkzh79iyPPPIIbm5uLFq0CMMwWLFihX4yvw7XOifJycmMHz+ehIQEPDw82LZtG2vWrCEtLY3Fixfj7q7nxa9Hc7jLz89n5cqVfPe73yUqKgp/f3+mTZsGwNixYwFYv369vd/x48e59957CQwMZNq0aZw5c4aFCxfSq1cvreLQAa5lXoqLi5k0aRITJkygT58+9lVotmzZwvjx43n11VcdczA3gBdeeIHFixdz++23c/fdd7f4ztceQc65AAAGEklEQVTXlzvuuAPQdaUrXc+c6LrSeTIzM/H09CQ1NRWr1crx48fJysqitLSUV155hfHjxwPOfa4owDtQfX09r732Gh9++CHV1dUkJibyk5/8hJEjR9q3aesfDzT9unnOnDls2rQJm83G8OHDee6554iOju7qw7ihXOuc/PznP2fnzp0cP36cxsZGIiMjGT9+PI899pge/uoAiYmJbbZHRkbag2FbAR7g4MGD/Pa3v2XHjh1YLBbGjBnD7NmzVarRAa5lXmpqavj1r39NTk4O5eXl2Gw2evfuzaRJk8jMzNRzCdeh+b9Nbbl4TnRd6TrXMye6rnSeZcuWkZ2dzaFDh6ipqaFHjx6kpKTw6KOPMmzYMPt2znyuKMCLiIiIiLgQ1cCLiIiIiLgQBXgREREREReiAC8iIiIi4kIU4EVEREREXIgCvIiIiIiIC1GAFxERERFxIQrwIiIiIiIuRAFeRESc3vTp0+0vhRIR6e70Hl4RkW5q27ZtZGZmtvu9m5sbeXl5XTgiERG5EgrwIiLd3IQJE7j11ltbtZvN+iWtiIgzUoAXEenmBgwYwMSJEx09DBERuUK6vSIiIpdUXFxMYmIi8+fPZ+XKlXznO99h8ODBjBkzhvnz53Pu3LlWffbt28dTTz3F8OHDGTx4MOPHj+ett97i/PnzrbatqKjgN7/5DePGjWPQoEGkp6fzyCOPsGnTplbblpWV8ZOf/ISbb76Z5ORkfvCDH1BQUNApxy0i4qx0B15EpJs7e/YsJ0+ebNXu4eGBn5+f/fP69espKirioYceIiQkhPXr1/OHP/yBkpISXnzxRft2X375JdOnT8fd3d2+7YYNG5g7dy779u3j5Zdftm9bXFzM1KlTqaysZOLEiQwaNIizZ8+Sk5PD5s2bueWWW+zbnjlzhmnTppGcnMzMmTMpLi5m8eLFPPnkk6xcuRI3N7dO+hsSEXEuCvAiIt3c/PnzmT9/fqv2MWPG8MYbb9g/79u3j2XLljFw4EAApk2bxtNPP01WVhZTpkwhJSUFgBdeeIGGhgbef/99kpKS7Ns+88wzrFy5kvvvv5/09HQAfvWrX1FeXs6CBQsYPXp0i/3bbLYWn7/66it+8IMfMGPGDHtbUFAQL730Eps3b27VX0TkRqUALyLSzU2ZMoWMjIxW7UFBQS0+jxw50h7eAUwmEz/84Q9Zt24dH3/8MSkpKVRWVrJr1y7uvPNOe3hv3vaJJ55g9erVfPzxx6Snp1NVVcW//vUvRo8e3Wb4/uZDtGazudWqOSNGjADg6NGjCvAi0m0owIuIdHOxsbGMHDnystvFx8e3auvbty8ARUVFQFNJzMXtF+vTpw9ms9m+bWFhIYZhMGDAgCsaZ2hoKJ6eni3aAgMDAaiqqrqiP0NE5Eagh1hFRMQlXKrG3TCMLhyJiIhjKcCLiMgVyc/Pb9V26NAhAKKjowGIiopq0X6xw4cPY7PZ7NvGxMRgMpnYu3dvZw1ZROSGpAAvIiJXZPPmzezZs8f+2TAMFixYAMAdd9wBQHBwMKmpqWzYsIEDBw602PbNN98E4M477wSayl9uvfVWNm7cyObNm1vtT3fVRUTaphp4EZFuLi8vj+zs7Da/aw7mAElJSTz88MM89NBDWK1WPvnkEzZv3szEiRNJTU21b/fcc88xffp0HnroIR588EGsVisbNmzgs88+Y8KECfYVaAB+8YtfkJeXx4wZM7j33nsZOHAg9fX15OTkEBkZyX//93933oGLiLgoBXgRkW5u5cqVrFy5ss3v1q5da689Hzt2LHFxcbzxxhsUFBQQHBzMk08+yZNPPtmiz+DBg3n//ff5/e9/z1/+8hfOnDlDdHQ0P/3pT3n00UdbbBsdHc3f/vY3/vjHP7Jx40ays7Px9/cnKSmJKVOmdM4Bi4i4OJOh31GKiMglFBcXM27cOJ5++mn+4z/+w9HDERHp9lQDLyIiIiLiQhTgRURERERciAK8iIiIiIgLUQ28iIiIiIgL0R14EREREREXogAvIiIiIuJCFOBFRERERFyIAryIiIiIiAtRgBcRERERcSEK8CIiIiIiLuT/A3Ug7tbNdNXYAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NBm8rA6y1jyU",
        "outputId": "ea050d34-622b-45ba-b295-faf38d659222",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# Test Dataset!\n",
        "\n",
        "import pandas as pd\n",
        "\n",
        "# Load the dataset into a pandas dataframe.\n",
        "#df = pd.read_csv(\"./cola_public/raw/out_of_domain_dev.tsv\", delimiter='\\t', header=None, names=['sentence_source', 'label', 'label_notes', 'sentence'])\n",
        "df= test\n",
        "# Report the number of sentences.\n",
        "print('Number of test sentences: {:,}\\n'.format(df.shape[0]))\n",
        "\n",
        "# Create sentence and label lists\n",
        "sentences = df.tweet.values\n",
        "labels = df.label.values\n",
        "\n",
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "input_ids = []\n",
        "\n",
        "# For every sentence...\n",
        "for sent in sentences:\n",
        "    # `encode` will:\n",
        "    #   (1) Tokenize the sentence.\n",
        "    #   (2) Prepend the `[CLS]` token to the start.\n",
        "    #   (3) Append the `[SEP]` token to the end.\n",
        "    #   (4) Map tokens to their IDs.\n",
        "    encoded_sent = tokenizer.encode(\n",
        "                        sent,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                   )\n",
        "    \n",
        "    input_ids.append(encoded_sent)\n",
        "\n",
        "# Pad our input tokens\n",
        "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, \n",
        "                          dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "\n",
        "# Create attention masks\n",
        "attention_masks = []\n",
        "\n",
        "# Create a mask of 1s for each token followed by 0s for padding\n",
        "for seq in input_ids:\n",
        "  seq_mask = [float(i>0) for i in seq]\n",
        "  attention_masks.append(seq_mask) \n",
        "\n",
        "# Convert to tensors.\n",
        "prediction_inputs = torch.tensor(input_ids)\n",
        "prediction_masks = torch.tensor(attention_masks)\n",
        "prediction_labels = torch.tensor(labels)\n",
        "\n",
        "# Set the batch size.  \n",
        "batch_size = 4 \n",
        "\n",
        "# Create the DataLoader.\n",
        "prediction_data = TensorDataset(prediction_inputs, prediction_masks, prediction_labels)\n",
        "prediction_sampler = SequentialSampler(prediction_data)\n",
        "prediction_dataloader = DataLoader(prediction_data, \n",
        "                                   sampler=prediction_sampler,\n",
        "                                   batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of test sentences: 1,000\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kmpYuvZn03hi"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "def softmax(x):\n",
        "\n",
        "    e_x = np.exp(x - np.max(x))\n",
        "\n",
        "    return e_x / e_x.sum()\n",
        "\n",
        "from scipy.special import softmax    \n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WalBbo9eIi1W",
        "outputId": "e9414762-ce13-436a-f1ba-f01392962253",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# Prediction on test set\n",
        "import numpy as np\n",
        "\n",
        "print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs)))\n",
        "\n",
        "# Put model in evaluation mode\n",
        "model.eval()\n",
        "\n",
        "# Tracking variables \n",
        "predictions , true_labels = [], []\n",
        "\n",
        "# Predict \n",
        "# y= []\n",
        "y= pd.DataFrame()\n",
        "for batch in prediction_dataloader:\n",
        "  # Add batch to GPU\n",
        "  batch = tuple(t.to(device) for t in batch)\n",
        "  \n",
        "  # Unpack the inputs from our dataloader\n",
        "  b_input_ids, b_input_mask, b_labels = batch\n",
        "  \n",
        "  # Telling the model not to compute or store gradients, saving memory and \n",
        "  # speeding up prediction\n",
        "  with torch.no_grad():\n",
        "      # Forward pass, calculate logit predictions\n",
        "      outputs = model(b_input_ids, token_type_ids=None, \n",
        "                      attention_mask=b_input_mask)\n",
        "\n",
        "  logits = outputs[0]\n",
        "\n",
        "  # Multiple predictions\n",
        "  pred=[]\n",
        "  collect=[]\n",
        "  p=[]\n",
        "  #w=[]\n",
        "  w = pd.DataFrame()\n",
        "\n",
        "\n",
        "\n",
        "  for _ in range(1000):\n",
        "            model.train()\n",
        "            pred = model(b_input_ids, token_type_ids=None, \n",
        "                      attention_mask=b_input_mask)\n",
        "            p=pred[0].detach().cpu().numpy()\n",
        "            x = np.reshape(p, (4, 2))\n",
        "            x = pd.DataFrame(x)\n",
        "            x2 = softmax(x, axis=1)\n",
        "            x3 = x2.drop([0], axis=1)\n",
        "            w = pd.concat([w, x3], axis=1, ignore_index=True)\n",
        "            #collect.append(x3)\n",
        "\n",
        "            # collect.append(np.matrix(p)\n",
        "\n",
        "#   x = np.reshape(collect, (40, 2))\n",
        "#   x = pd.DataFrame(x)\n",
        "#   x2 = softmax(x, axis=1)\n",
        "#   x3 = x2.drop([1], axis=1)\n",
        "  w = pd.DataFrame(w)\n",
        "  y = y.append(w)\n",
        "\n",
        "  \n",
        "  # Move logits and labels to CPU\n",
        "  logits = logits.detach().cpu().numpy()\n",
        "  label_ids = b_labels.to('cpu').numpy()\n",
        "  \n",
        "  # Store predictions and true labels\n",
        "  predictions.append(logits)\n",
        "  true_labels.append(label_ids)\n",
        "\n",
        "print('    DONE.')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predicting labels for 1,000 test sentences...\n",
            "    DONE.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yP-TiLprciui"
      },
      "source": [
        "y.head()\n",
        "y=y.reset_index(drop=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1U4pHG-FiyVX",
        "outputId": "533f17e3-dcaa-44c7-9c8f-af8211662c42",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "flat_true_labels = [item for sublist in true_labels for item in sublist]\n",
        "flat_true_labels"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 1,\n",
              " 1,\n",
              " 0,\n",
              " 1,\n",
              " 0]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LMxN3VHTuxJM",
        "outputId": "ab4f66ba-22d0-47ee-f200-837ab93e26df",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        }
      },
      "source": [
        "#MC_pred = test.reshape(100,1000)\n",
        "#pred=pd.DataFrame(MC_pred)\n",
        "means=y.mean(axis=1)\n",
        "type(means)\n",
        "print(len(means))\n",
        "print(means)\n",
        "\n",
        "# To make histograms with the pred values and y_test"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1000\n",
            "0      0.000591\n",
            "1      0.000452\n",
            "2      0.000955\n",
            "3      0.000838\n",
            "4      0.999620\n",
            "         ...   \n",
            "995    0.999680\n",
            "996    0.999333\n",
            "997    0.001274\n",
            "998    0.999479\n",
            "999    0.665915\n",
            "Length: 1000, dtype: float32\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "39lvPuvWjzRX"
      },
      "source": [
        "test = pd.DataFrame(test)\n",
        "\n",
        "test=test.reset_index(drop=True)\n",
        "means= pd.DataFrame(means)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7OQmp574tlOA",
        "outputId": "1e582d9c-64e2-45e2-e9cc-ef1e30834042",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "text=test['tweet']\n",
        "text= pd.DataFrame(text)\n",
        "text=text.reset_index(drop=True)\n",
        "text.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XOEg6KyxjfQV",
        "outputId": "fa9e15a2-4a64-45fa-9f40-5679c272d600",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "test.shape\n",
        "y.shape\n",
        "type(y)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "pandas.core.frame.DataFrame"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LsA_mfuTjkvJ"
      },
      "source": [
        "result = pd.concat([means, test,y], axis=1, sort=False)\n",
        "#result = pd.concat([test, y], axis=1, ignore_index=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VNkYVsREvMzq",
        "outputId": "430dc5c0-4198-476c-d819-42e063dc0dd9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 508
        }
      },
      "source": [
        "result.tail()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>id</th>\n",
              "      <th>label</th>\n",
              "      <th>tweet</th>\n",
              "      <th>normalized_text</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>...</th>\n",
              "      <th>960</th>\n",
              "      <th>961</th>\n",
              "      <th>962</th>\n",
              "      <th>963</th>\n",
              "      <th>964</th>\n",
              "      <th>965</th>\n",
              "      <th>966</th>\n",
              "      <th>967</th>\n",
              "      <th>968</th>\n",
              "      <th>969</th>\n",
              "      <th>970</th>\n",
              "      <th>971</th>\n",
              "      <th>972</th>\n",
              "      <th>973</th>\n",
              "      <th>974</th>\n",
              "      <th>975</th>\n",
              "      <th>976</th>\n",
              "      <th>977</th>\n",
              "      <th>978</th>\n",
              "      <th>979</th>\n",
              "      <th>980</th>\n",
              "      <th>981</th>\n",
              "      <th>982</th>\n",
              "      <th>983</th>\n",
              "      <th>984</th>\n",
              "      <th>985</th>\n",
              "      <th>986</th>\n",
              "      <th>987</th>\n",
              "      <th>988</th>\n",
              "      <th>989</th>\n",
              "      <th>990</th>\n",
              "      <th>991</th>\n",
              "      <th>992</th>\n",
              "      <th>993</th>\n",
              "      <th>994</th>\n",
              "      <th>995</th>\n",
              "      <th>996</th>\n",
              "      <th>997</th>\n",
              "      <th>998</th>\n",
              "      <th>999</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>995</th>\n",
              "      <td>0.999680</td>\n",
              "      <td>3475</td>\n",
              "      <td>1</td>\n",
              "      <td>sea shepherd suppoers are racist!   #antiraci...</td>\n",
              "      <td>sea shepherd suppoers racist antiracism seashe...</td>\n",
              "      <td>0.999778</td>\n",
              "      <td>0.999691</td>\n",
              "      <td>0.999654</td>\n",
              "      <td>0.999599</td>\n",
              "      <td>0.999685</td>\n",
              "      <td>0.999643</td>\n",
              "      <td>0.999758</td>\n",
              "      <td>0.999711</td>\n",
              "      <td>0.999741</td>\n",
              "      <td>0.999659</td>\n",
              "      <td>0.999748</td>\n",
              "      <td>0.999724</td>\n",
              "      <td>0.999746</td>\n",
              "      <td>0.999517</td>\n",
              "      <td>0.999653</td>\n",
              "      <td>0.999774</td>\n",
              "      <td>0.999690</td>\n",
              "      <td>0.999646</td>\n",
              "      <td>0.999734</td>\n",
              "      <td>0.999681</td>\n",
              "      <td>0.999716</td>\n",
              "      <td>0.999704</td>\n",
              "      <td>0.999667</td>\n",
              "      <td>0.999642</td>\n",
              "      <td>0.999655</td>\n",
              "      <td>0.999698</td>\n",
              "      <td>0.999666</td>\n",
              "      <td>0.999668</td>\n",
              "      <td>0.999680</td>\n",
              "      <td>0.999646</td>\n",
              "      <td>0.999648</td>\n",
              "      <td>0.999705</td>\n",
              "      <td>0.999634</td>\n",
              "      <td>0.999692</td>\n",
              "      <td>0.999700</td>\n",
              "      <td>...</td>\n",
              "      <td>0.999748</td>\n",
              "      <td>0.999739</td>\n",
              "      <td>0.999498</td>\n",
              "      <td>0.999716</td>\n",
              "      <td>0.999584</td>\n",
              "      <td>0.999696</td>\n",
              "      <td>0.999690</td>\n",
              "      <td>0.999757</td>\n",
              "      <td>0.999551</td>\n",
              "      <td>0.999747</td>\n",
              "      <td>0.999723</td>\n",
              "      <td>0.999727</td>\n",
              "      <td>0.999748</td>\n",
              "      <td>0.999778</td>\n",
              "      <td>0.999698</td>\n",
              "      <td>0.999732</td>\n",
              "      <td>0.999697</td>\n",
              "      <td>0.999743</td>\n",
              "      <td>0.999691</td>\n",
              "      <td>0.999608</td>\n",
              "      <td>0.999748</td>\n",
              "      <td>0.999705</td>\n",
              "      <td>0.999637</td>\n",
              "      <td>0.999723</td>\n",
              "      <td>0.999700</td>\n",
              "      <td>0.999665</td>\n",
              "      <td>0.999561</td>\n",
              "      <td>0.999676</td>\n",
              "      <td>0.999691</td>\n",
              "      <td>0.999731</td>\n",
              "      <td>0.999694</td>\n",
              "      <td>0.999735</td>\n",
              "      <td>0.999706</td>\n",
              "      <td>0.999731</td>\n",
              "      <td>0.999627</td>\n",
              "      <td>0.999709</td>\n",
              "      <td>0.999598</td>\n",
              "      <td>0.999555</td>\n",
              "      <td>0.999712</td>\n",
              "      <td>0.999689</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996</th>\n",
              "      <td>0.999333</td>\n",
              "      <td>14822</td>\n",
              "      <td>1</td>\n",
              "      <td>@user #mediamisogyny, russian interference,  c...</td>\n",
              "      <td>mediamisogyny russian interference crosschecki...</td>\n",
              "      <td>0.999494</td>\n",
              "      <td>0.999556</td>\n",
              "      <td>0.999339</td>\n",
              "      <td>0.999259</td>\n",
              "      <td>0.999508</td>\n",
              "      <td>0.999493</td>\n",
              "      <td>0.999086</td>\n",
              "      <td>0.999088</td>\n",
              "      <td>0.999326</td>\n",
              "      <td>0.999461</td>\n",
              "      <td>0.999144</td>\n",
              "      <td>0.998785</td>\n",
              "      <td>0.999536</td>\n",
              "      <td>0.998790</td>\n",
              "      <td>0.999311</td>\n",
              "      <td>0.999554</td>\n",
              "      <td>0.999453</td>\n",
              "      <td>0.999352</td>\n",
              "      <td>0.999401</td>\n",
              "      <td>0.999544</td>\n",
              "      <td>0.999494</td>\n",
              "      <td>0.999441</td>\n",
              "      <td>0.999098</td>\n",
              "      <td>0.999568</td>\n",
              "      <td>0.999534</td>\n",
              "      <td>0.999455</td>\n",
              "      <td>0.999444</td>\n",
              "      <td>0.999428</td>\n",
              "      <td>0.999189</td>\n",
              "      <td>0.999596</td>\n",
              "      <td>0.999205</td>\n",
              "      <td>0.999523</td>\n",
              "      <td>0.999306</td>\n",
              "      <td>0.999210</td>\n",
              "      <td>0.999316</td>\n",
              "      <td>...</td>\n",
              "      <td>0.999439</td>\n",
              "      <td>0.999392</td>\n",
              "      <td>0.999551</td>\n",
              "      <td>0.999427</td>\n",
              "      <td>0.999470</td>\n",
              "      <td>0.999424</td>\n",
              "      <td>0.999547</td>\n",
              "      <td>0.999163</td>\n",
              "      <td>0.999290</td>\n",
              "      <td>0.999479</td>\n",
              "      <td>0.999434</td>\n",
              "      <td>0.999373</td>\n",
              "      <td>0.999437</td>\n",
              "      <td>0.999589</td>\n",
              "      <td>0.998950</td>\n",
              "      <td>0.999081</td>\n",
              "      <td>0.999084</td>\n",
              "      <td>0.999341</td>\n",
              "      <td>0.999581</td>\n",
              "      <td>0.999611</td>\n",
              "      <td>0.998722</td>\n",
              "      <td>0.999593</td>\n",
              "      <td>0.999547</td>\n",
              "      <td>0.999423</td>\n",
              "      <td>0.999215</td>\n",
              "      <td>0.999386</td>\n",
              "      <td>0.999207</td>\n",
              "      <td>0.999439</td>\n",
              "      <td>0.999512</td>\n",
              "      <td>0.998556</td>\n",
              "      <td>0.999299</td>\n",
              "      <td>0.999216</td>\n",
              "      <td>0.999285</td>\n",
              "      <td>0.999391</td>\n",
              "      <td>0.999405</td>\n",
              "      <td>0.999352</td>\n",
              "      <td>0.998966</td>\n",
              "      <td>0.999223</td>\n",
              "      <td>0.999414</td>\n",
              "      <td>0.999458</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>997</th>\n",
              "      <td>0.001274</td>\n",
              "      <td>30394</td>\n",
              "      <td>0</td>\n",
              "      <td>i'm so   and #grateful now that - #affirmations</td>\n",
              "      <td>grateful affirmation</td>\n",
              "      <td>0.001357</td>\n",
              "      <td>0.001515</td>\n",
              "      <td>0.001132</td>\n",
              "      <td>0.001143</td>\n",
              "      <td>0.001617</td>\n",
              "      <td>0.001083</td>\n",
              "      <td>0.001403</td>\n",
              "      <td>0.000862</td>\n",
              "      <td>0.002081</td>\n",
              "      <td>0.001580</td>\n",
              "      <td>0.001141</td>\n",
              "      <td>0.000857</td>\n",
              "      <td>0.000866</td>\n",
              "      <td>0.001532</td>\n",
              "      <td>0.000814</td>\n",
              "      <td>0.000607</td>\n",
              "      <td>0.001526</td>\n",
              "      <td>0.001158</td>\n",
              "      <td>0.001548</td>\n",
              "      <td>0.001357</td>\n",
              "      <td>0.001026</td>\n",
              "      <td>0.001089</td>\n",
              "      <td>0.002376</td>\n",
              "      <td>0.000695</td>\n",
              "      <td>0.001823</td>\n",
              "      <td>0.000880</td>\n",
              "      <td>0.001875</td>\n",
              "      <td>0.001057</td>\n",
              "      <td>0.000864</td>\n",
              "      <td>0.000831</td>\n",
              "      <td>0.001157</td>\n",
              "      <td>0.000970</td>\n",
              "      <td>0.001697</td>\n",
              "      <td>0.001109</td>\n",
              "      <td>0.000882</td>\n",
              "      <td>...</td>\n",
              "      <td>0.001189</td>\n",
              "      <td>0.000682</td>\n",
              "      <td>0.001414</td>\n",
              "      <td>0.002152</td>\n",
              "      <td>0.000886</td>\n",
              "      <td>0.002051</td>\n",
              "      <td>0.001543</td>\n",
              "      <td>0.002172</td>\n",
              "      <td>0.001019</td>\n",
              "      <td>0.001420</td>\n",
              "      <td>0.001110</td>\n",
              "      <td>0.001475</td>\n",
              "      <td>0.002272</td>\n",
              "      <td>0.000888</td>\n",
              "      <td>0.001115</td>\n",
              "      <td>0.001063</td>\n",
              "      <td>0.000818</td>\n",
              "      <td>0.002345</td>\n",
              "      <td>0.001227</td>\n",
              "      <td>0.000757</td>\n",
              "      <td>0.001299</td>\n",
              "      <td>0.000711</td>\n",
              "      <td>0.001163</td>\n",
              "      <td>0.000751</td>\n",
              "      <td>0.000917</td>\n",
              "      <td>0.001195</td>\n",
              "      <td>0.001433</td>\n",
              "      <td>0.001168</td>\n",
              "      <td>0.001419</td>\n",
              "      <td>0.001128</td>\n",
              "      <td>0.000984</td>\n",
              "      <td>0.003167</td>\n",
              "      <td>0.000852</td>\n",
              "      <td>0.000713</td>\n",
              "      <td>0.001178</td>\n",
              "      <td>0.001409</td>\n",
              "      <td>0.001455</td>\n",
              "      <td>0.001512</td>\n",
              "      <td>0.000942</td>\n",
              "      <td>0.001488</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>998</th>\n",
              "      <td>0.999479</td>\n",
              "      <td>16523</td>\n",
              "      <td>1</td>\n",
              "      <td>why are white people #expats when the rest of ...</td>\n",
              "      <td>white people expat rest u immigrant socialclas...</td>\n",
              "      <td>0.999662</td>\n",
              "      <td>0.999587</td>\n",
              "      <td>0.999660</td>\n",
              "      <td>0.999635</td>\n",
              "      <td>0.999684</td>\n",
              "      <td>0.999635</td>\n",
              "      <td>0.999473</td>\n",
              "      <td>0.999536</td>\n",
              "      <td>0.999522</td>\n",
              "      <td>0.999557</td>\n",
              "      <td>0.999395</td>\n",
              "      <td>0.999497</td>\n",
              "      <td>0.999534</td>\n",
              "      <td>0.999509</td>\n",
              "      <td>0.999529</td>\n",
              "      <td>0.999335</td>\n",
              "      <td>0.999440</td>\n",
              "      <td>0.999368</td>\n",
              "      <td>0.999569</td>\n",
              "      <td>0.999533</td>\n",
              "      <td>0.999711</td>\n",
              "      <td>0.999410</td>\n",
              "      <td>0.999504</td>\n",
              "      <td>0.999364</td>\n",
              "      <td>0.999554</td>\n",
              "      <td>0.999565</td>\n",
              "      <td>0.999441</td>\n",
              "      <td>0.999533</td>\n",
              "      <td>0.999598</td>\n",
              "      <td>0.999681</td>\n",
              "      <td>0.999460</td>\n",
              "      <td>0.999523</td>\n",
              "      <td>0.999601</td>\n",
              "      <td>0.999529</td>\n",
              "      <td>0.999558</td>\n",
              "      <td>...</td>\n",
              "      <td>0.999482</td>\n",
              "      <td>0.999373</td>\n",
              "      <td>0.999301</td>\n",
              "      <td>0.999487</td>\n",
              "      <td>0.999533</td>\n",
              "      <td>0.999599</td>\n",
              "      <td>0.999234</td>\n",
              "      <td>0.999571</td>\n",
              "      <td>0.999391</td>\n",
              "      <td>0.999457</td>\n",
              "      <td>0.999594</td>\n",
              "      <td>0.999578</td>\n",
              "      <td>0.999225</td>\n",
              "      <td>0.999292</td>\n",
              "      <td>0.999391</td>\n",
              "      <td>0.999054</td>\n",
              "      <td>0.999472</td>\n",
              "      <td>0.999539</td>\n",
              "      <td>0.999636</td>\n",
              "      <td>0.999346</td>\n",
              "      <td>0.999633</td>\n",
              "      <td>0.999562</td>\n",
              "      <td>0.999551</td>\n",
              "      <td>0.999524</td>\n",
              "      <td>0.999441</td>\n",
              "      <td>0.999519</td>\n",
              "      <td>0.999544</td>\n",
              "      <td>0.999488</td>\n",
              "      <td>0.999697</td>\n",
              "      <td>0.999592</td>\n",
              "      <td>0.999250</td>\n",
              "      <td>0.999481</td>\n",
              "      <td>0.999491</td>\n",
              "      <td>0.999388</td>\n",
              "      <td>0.999595</td>\n",
              "      <td>0.999498</td>\n",
              "      <td>0.999682</td>\n",
              "      <td>0.999118</td>\n",
              "      <td>0.999480</td>\n",
              "      <td>0.999552</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999</th>\n",
              "      <td>0.665915</td>\n",
              "      <td>26635</td>\n",
              "      <td>0</td>\n",
              "      <td>giving away a preorder of full of beans at our...</td>\n",
              "      <td>give away preorder full bean nerdcampmi presen...</td>\n",
              "      <td>0.004946</td>\n",
              "      <td>0.688600</td>\n",
              "      <td>0.011032</td>\n",
              "      <td>0.206057</td>\n",
              "      <td>0.998708</td>\n",
              "      <td>0.071075</td>\n",
              "      <td>0.054299</td>\n",
              "      <td>0.002296</td>\n",
              "      <td>0.009928</td>\n",
              "      <td>0.990999</td>\n",
              "      <td>0.002690</td>\n",
              "      <td>0.998726</td>\n",
              "      <td>0.999226</td>\n",
              "      <td>0.004188</td>\n",
              "      <td>0.000896</td>\n",
              "      <td>0.999084</td>\n",
              "      <td>0.002647</td>\n",
              "      <td>0.943574</td>\n",
              "      <td>0.053614</td>\n",
              "      <td>0.011119</td>\n",
              "      <td>0.994492</td>\n",
              "      <td>0.998284</td>\n",
              "      <td>0.999184</td>\n",
              "      <td>0.004980</td>\n",
              "      <td>0.976058</td>\n",
              "      <td>0.998702</td>\n",
              "      <td>0.001393</td>\n",
              "      <td>0.003498</td>\n",
              "      <td>0.006810</td>\n",
              "      <td>0.998620</td>\n",
              "      <td>0.004735</td>\n",
              "      <td>0.426210</td>\n",
              "      <td>0.016976</td>\n",
              "      <td>0.975802</td>\n",
              "      <td>0.882351</td>\n",
              "      <td>...</td>\n",
              "      <td>0.976786</td>\n",
              "      <td>0.999515</td>\n",
              "      <td>0.999433</td>\n",
              "      <td>0.840628</td>\n",
              "      <td>0.982995</td>\n",
              "      <td>0.950006</td>\n",
              "      <td>0.998052</td>\n",
              "      <td>0.994707</td>\n",
              "      <td>0.999156</td>\n",
              "      <td>0.150277</td>\n",
              "      <td>0.995196</td>\n",
              "      <td>0.962573</td>\n",
              "      <td>0.074773</td>\n",
              "      <td>0.971148</td>\n",
              "      <td>0.999033</td>\n",
              "      <td>0.015908</td>\n",
              "      <td>0.997632</td>\n",
              "      <td>0.999091</td>\n",
              "      <td>0.997962</td>\n",
              "      <td>0.086627</td>\n",
              "      <td>0.998861</td>\n",
              "      <td>0.004627</td>\n",
              "      <td>0.998251</td>\n",
              "      <td>0.002237</td>\n",
              "      <td>0.001338</td>\n",
              "      <td>0.997935</td>\n",
              "      <td>0.976085</td>\n",
              "      <td>0.544352</td>\n",
              "      <td>0.002201</td>\n",
              "      <td>0.999360</td>\n",
              "      <td>0.999305</td>\n",
              "      <td>0.999406</td>\n",
              "      <td>0.011267</td>\n",
              "      <td>0.999075</td>\n",
              "      <td>0.993480</td>\n",
              "      <td>0.997685</td>\n",
              "      <td>0.943445</td>\n",
              "      <td>0.995342</td>\n",
              "      <td>0.002017</td>\n",
              "      <td>0.998238</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 1005 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "            0     id  label  ...       997       998       999\n",
              "995  0.999680   3475      1  ...  0.999555  0.999712  0.999689\n",
              "996  0.999333  14822      1  ...  0.999223  0.999414  0.999458\n",
              "997  0.001274  30394      0  ...  0.001512  0.000942  0.001488\n",
              "998  0.999479  16523      1  ...  0.999118  0.999480  0.999552\n",
              "999  0.665915  26635      0  ...  0.995342  0.002017  0.998238\n",
              "\n",
              "[5 rows x 1005 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2oF3PEqEzWFs"
      },
      "source": [
        "result.to_csv(r'/content/gdrive/My Drive/pred_BERT500.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Rpa_syxkNa8",
        "outputId": "6d9678dd-5543-4d44-e919-0fba45f9bc0d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "result.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 1005)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_rBHoSDjUktL"
      },
      "source": [
        "means[means >= 0.5] = 1\n",
        "means[means < 0.5] = 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HTPT0e-KVBNU",
        "outputId": "959e5e69-88a8-4ea9-acd4-bdb6017bbb50",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        }
      },
      "source": [
        "print(means)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "       0\n",
            "0    0.0\n",
            "1    0.0\n",
            "2    0.0\n",
            "3    0.0\n",
            "4    1.0\n",
            "..   ...\n",
            "995  1.0\n",
            "996  1.0\n",
            "997  0.0\n",
            "998  1.0\n",
            "999  1.0\n",
            "\n",
            "[1000 rows x 1 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FzH-VKi1TmFt",
        "outputId": "08841d2b-394b-4121-9d9f-52dff3ac3b6b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "from sklearn import metrics\n",
        "import numpy as np\n",
        "print(\"ACC:\",metrics.accuracy_score(flat_true_labels, means))\n",
        "print(\"PRE:\",metrics.precision_score(flat_true_labels, means))\n",
        "print(\"REC:\",metrics.recall_score(flat_true_labels, means))\n",
        "print(\"F1:\", metrics.f1_score(flat_true_labels, means))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ACC: 0.911\n",
            "PRE: 0.8949579831932774\n",
            "REC: 0.9161290322580645\n",
            "F1: 0.9054197662061636\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}