learning-new-tokens-loss.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Learn-New-Tokens-with-Metrics.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "183fd8b8da7a420da59475474e826c7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d3a61fa2a9c0437f8e5786bafafc97e9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1fe3733155be4201b4a631628e6f387c",
              "IPY_MODEL_e89139d5d03b4502ae5e16a4891fb1f2",
              "IPY_MODEL_2db449311da94c3fb53d774e9a0ee91a"
            ]
          }
        },
        "d3a61fa2a9c0437f8e5786bafafc97e9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1fe3733155be4201b4a631628e6f387c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b1936d69238f4534a77ebc613c496daa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_edcd1d485a2f42bcac26805b8e7fca57"
          }
        },
        "e89139d5d03b4502ae5e16a4891fb1f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_929c39ea765e45e8a88e460e6e21128f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_22c014b4f7bf4cbda3a91b7f086ba59c"
          }
        },
        "2db449311da94c3fb53d774e9a0ee91a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9ee60a02ee044e59bc53753477cd359b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:00&lt;00:00, 23.25it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_03352d80fc48434ebc26052e16cb16e4"
          }
        },
        "b1936d69238f4534a77ebc613c496daa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "edcd1d485a2f42bcac26805b8e7fca57": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "929c39ea765e45e8a88e460e6e21128f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "22c014b4f7bf4cbda3a91b7f086ba59c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9ee60a02ee044e59bc53753477cd359b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "03352d80fc48434ebc26052e16cb16e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "618ddd39e1f34711b7c8868f76d1b763": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_349b3234095643d486d13e659a00d136",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b4d9902c8c2147ba861183e1e50c46f5",
              "IPY_MODEL_50944cda66f14ab78d0a15262cfc1bd1",
              "IPY_MODEL_1eef083e7e8e4247b4cb6be0dd64a570"
            ]
          }
        },
        "349b3234095643d486d13e659a00d136": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b4d9902c8c2147ba861183e1e50c46f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fdfd03df54304f10a11edb6ca840bb67",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0be3581e068244ffac134abbebd903ca"
          }
        },
        "50944cda66f14ab78d0a15262cfc1bd1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a2c1d424ad29476a913bf4f4868b7b2b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8de6c28c128e4ebc87ba171ff625895f"
          }
        },
        "1eef083e7e8e4247b4cb6be0dd64a570": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_48f5fa53d94f4ff9a10a5ca626ce2396",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:00&lt;00:00, 23.84it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5259a1826ff34a5185968dcf073bc205"
          }
        },
        "fdfd03df54304f10a11edb6ca840bb67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0be3581e068244ffac134abbebd903ca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a2c1d424ad29476a913bf4f4868b7b2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8de6c28c128e4ebc87ba171ff625895f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "48f5fa53d94f4ff9a10a5ca626ce2396": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5259a1826ff34a5185968dcf073bc205": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ed39797f16b841809a8e55f2eed8f3e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3a25defe466d4a0fb31110a04f4ea46e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9f39466a86134e63bfa8fe3690dd4bc0",
              "IPY_MODEL_fb1416c8cbc747eea07bd5319e4b53e7",
              "IPY_MODEL_ea55a4bb0a1f442a90b83fb1dc1a97e9"
            ]
          }
        },
        "3a25defe466d4a0fb31110a04f4ea46e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9f39466a86134e63bfa8fe3690dd4bc0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d8bd1b83ee01468d98fbea5dedef0639",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8dbd18575f1a4090bbaab75bd9cc710b"
          }
        },
        "fb1416c8cbc747eea07bd5319e4b53e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8a12ae8fade549008d0f9a125a4d4b56",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_861a6feb84a14864b841a5d9322e668a"
          }
        },
        "ea55a4bb0a1f442a90b83fb1dc1a97e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_31b8aad9d82c4147b2c20834de59fbfa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:00&lt;00:00, 24.41it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2814618811de4f23863d571292ac3039"
          }
        },
        "d8bd1b83ee01468d98fbea5dedef0639": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8dbd18575f1a4090bbaab75bd9cc710b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8a12ae8fade549008d0f9a125a4d4b56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "861a6feb84a14864b841a5d9322e668a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "31b8aad9d82c4147b2c20834de59fbfa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2814618811de4f23863d571292ac3039": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "159905c697814dd9b457c71096afd1f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e63469e16e1b463c8ff5bcad1a4657c6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3139f932c33e415780b248d14c1ff51c",
              "IPY_MODEL_68eaa842c14c4d69aa8a7b9dc3908769",
              "IPY_MODEL_38f876fef866469aac6ca4769b0206f2"
            ]
          }
        },
        "e63469e16e1b463c8ff5bcad1a4657c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3139f932c33e415780b248d14c1ff51c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1da3cde112914ce190ac3c802bb97f56",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_28ccbfaa32194b2a9d55ee4e63ab510b"
          }
        },
        "68eaa842c14c4d69aa8a7b9dc3908769": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e04d1cd1e6494b5eaba83008c472ff93",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 665,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 665,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1186b6992b3640ebac05e49ab7291efa"
          }
        },
        "38f876fef866469aac6ca4769b0206f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_73d8aae92f0b4fa185cbc61d834d2a3c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 665/665 [00:00&lt;00:00, 16.0kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_294556f9af424526a0e3760677af8715"
          }
        },
        "1da3cde112914ce190ac3c802bb97f56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "28ccbfaa32194b2a9d55ee4e63ab510b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e04d1cd1e6494b5eaba83008c472ff93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1186b6992b3640ebac05e49ab7291efa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "73d8aae92f0b4fa185cbc61d834d2a3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "294556f9af424526a0e3760677af8715": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "91719f65f2354310aeb45a5b06a28c6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_af071bc680da46c1a6038f0d6515bb86",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cfe4732816364b6da651d54ca131c972",
              "IPY_MODEL_5836da7037234081b106940dd083083c",
              "IPY_MODEL_0e036abea87f49afab4b60316914777e"
            ]
          }
        },
        "af071bc680da46c1a6038f0d6515bb86": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cfe4732816364b6da651d54ca131c972": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ee4fd5d947324f66a1109f1d084ec510",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9d995c8ad1c3418eb46db63771ecb68d"
          }
        },
        "5836da7037234081b106940dd083083c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_40ff380dfc994589ba78c75a0cecba74",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1042301,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1042301,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d5c9f5d1f4214bfeaf6e715dc9dd635a"
          }
        },
        "0e036abea87f49afab4b60316914777e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_569c40ac513249b9b8580d99fc36c817",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 0.99M/0.99M [00:01&lt;00:00, 1.18MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8b174b0125dd485482335d7a732b45d6"
          }
        },
        "ee4fd5d947324f66a1109f1d084ec510": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9d995c8ad1c3418eb46db63771ecb68d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "40ff380dfc994589ba78c75a0cecba74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d5c9f5d1f4214bfeaf6e715dc9dd635a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "569c40ac513249b9b8580d99fc36c817": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8b174b0125dd485482335d7a732b45d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8c6c7032705947c0b059fc1643e618df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6afbca38b20e4af9bb93d9ac82924e5a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_82bb714db3ed496980c2dff1ff304ffd",
              "IPY_MODEL_4413b6b8764045c6892038fe421da18a",
              "IPY_MODEL_14b1a59b71754471ae8114a3f5b29998"
            ]
          }
        },
        "6afbca38b20e4af9bb93d9ac82924e5a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "82bb714db3ed496980c2dff1ff304ffd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7b0dcbaefeed4c03a39bc70cd22ea634",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a77f9e75aa5f47e1b06e66ac9d46daad"
          }
        },
        "4413b6b8764045c6892038fe421da18a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_12fad5558d68406b8b24ed48e573d6d6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 456318,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 456318,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f0fe9a5e5a4d4a05ba71ad4675343af3"
          }
        },
        "14b1a59b71754471ae8114a3f5b29998": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f7387ccb857f44b2a71a348d09e99d13",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 446k/446k [00:00&lt;00:00, 538kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d32e5c0b242c4975982975978dd8b386"
          }
        },
        "7b0dcbaefeed4c03a39bc70cd22ea634": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a77f9e75aa5f47e1b06e66ac9d46daad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "12fad5558d68406b8b24ed48e573d6d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f0fe9a5e5a4d4a05ba71ad4675343af3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f7387ccb857f44b2a71a348d09e99d13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d32e5c0b242c4975982975978dd8b386": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b3cc9a6582444ed1b230ebe62903899d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_773e6f8aa2f64c738268561a9058ed85",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f53bd7d4842d40aeaa370fa4ead86147",
              "IPY_MODEL_18bf96a80c10445a8d0568e252951e30",
              "IPY_MODEL_ca88243771524d1c991f004a157c3a95"
            ]
          }
        },
        "773e6f8aa2f64c738268561a9058ed85": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f53bd7d4842d40aeaa370fa4ead86147": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b030e6a1b007481a8c37bbe4e4275fbf",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_68b50b59da1844d1863abb82f2ac5331"
          }
        },
        "18bf96a80c10445a8d0568e252951e30": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_33403267bff2447f84bbbda19f2915c3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1355256,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1355256,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0f4334b894b845cd8454bd673a9b986d"
          }
        },
        "ca88243771524d1c991f004a157c3a95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ee4e0206133a4351bbf2b922c5346dad",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.29M/1.29M [00:01&lt;00:00, 1.18MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6afb1a29518f472b963d1c65be76b54f"
          }
        },
        "b030e6a1b007481a8c37bbe4e4275fbf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "68b50b59da1844d1863abb82f2ac5331": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "33403267bff2447f84bbbda19f2915c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0f4334b894b845cd8454bd673a9b986d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ee4e0206133a4351bbf2b922c5346dad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6afb1a29518f472b963d1c65be76b54f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b96a1bcebf054f0581c8ccd4caa80127": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_758d09e5883c473e8bbdc6c9ec63cc33",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d5d3bcb4c20e48a9b8a42d808ee982df",
              "IPY_MODEL_0b00094ede694737b3d381c3f3405b4a",
              "IPY_MODEL_c74c64bcf8fa42f2a61f5ea7c6f87be3"
            ]
          }
        },
        "758d09e5883c473e8bbdc6c9ec63cc33": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d5d3bcb4c20e48a9b8a42d808ee982df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_428ba775d3584cf786d5d2f3700c6e61",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_79a540007d264607beb4db016b7faa67"
          }
        },
        "0b00094ede694737b3d381c3f3405b4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cde043d7cc414dba981e66353ddad925",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 548118077,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 548118077,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e26f9b3f6f34505bbec766f028a0bb5"
          }
        },
        "c74c64bcf8fa42f2a61f5ea7c6f87be3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_feddff02f44446538cd2625060a558f0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 523M/523M [00:16&lt;00:00, 34.8MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_eeaf0b8dd30a4aee9cc2620138cf79e9"
          }
        },
        "428ba775d3584cf786d5d2f3700c6e61": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "79a540007d264607beb4db016b7faa67": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cde043d7cc414dba981e66353ddad925": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e26f9b3f6f34505bbec766f028a0bb5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "feddff02f44446538cd2625060a558f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "eeaf0b8dd30a4aee9cc2620138cf79e9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bb1881cc120749978cf226fa908e1de2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a65003e7d3f24c48b6ced56f41631390",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c76e5749cc9c49738e669bd7130ba92c",
              "IPY_MODEL_1685b2a6b7f54fab94ea6a506b3ba211",
              "IPY_MODEL_d2fe9ed993fa4f11bad266c9b7242ec5"
            ]
          }
        },
        "a65003e7d3f24c48b6ced56f41631390": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c76e5749cc9c49738e669bd7130ba92c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bd4ea502336c4d8db0cb0a6675f69a91",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Running tokenizer on dataset: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_925b4da85dc34d1585540c822fd26474"
          }
        },
        "1685b2a6b7f54fab94ea6a506b3ba211": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cf568ab2a00944348d22b852744ead75",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 14,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 14,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a49288faa99846c3bace7aba46ba7361"
          }
        },
        "d2fe9ed993fa4f11bad266c9b7242ec5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1e7b9b87512648b5838dd5a9d5519d87",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 14/14 [03:05&lt;00:00, 11.29s/ba]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1840298801444267bc3f11e1b05cdcb6"
          }
        },
        "bd4ea502336c4d8db0cb0a6675f69a91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "925b4da85dc34d1585540c822fd26474": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cf568ab2a00944348d22b852744ead75": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a49288faa99846c3bace7aba46ba7361": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1e7b9b87512648b5838dd5a9d5519d87": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1840298801444267bc3f11e1b05cdcb6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "55b5a90e0e1941898916e20aca4ed5e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_dd0c646708164d89936106f437eec677",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3538feac33214a4d94e19bf33d64811c",
              "IPY_MODEL_6250a0ed184a4c86a82f66ffc72aea2e",
              "IPY_MODEL_fdffbbf4a50a4dcd9105405040cb9057"
            ]
          }
        },
        "dd0c646708164d89936106f437eec677": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3538feac33214a4d94e19bf33d64811c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6df45c6fef00459aa36b14beaf0a5c6b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Grouping texts in chunks of 1024: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6937a423b8cc42538ef771d4270e5eb5"
          }
        },
        "6250a0ed184a4c86a82f66ffc72aea2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_58eadf9845124173815a0684f356e9ba",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 14,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 14,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8804c0b5e6684f9da4d41c7a3c57404f"
          }
        },
        "fdffbbf4a50a4dcd9105405040cb9057": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5e56639357474563904c28e30f9d4d9b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 14/14 [00:18&lt;00:00,  1.08s/ba]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9776911d31b4438397a74f745291cafe"
          }
        },
        "6df45c6fef00459aa36b14beaf0a5c6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6937a423b8cc42538ef771d4270e5eb5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "58eadf9845124173815a0684f356e9ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8804c0b5e6684f9da4d41c7a3c57404f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5e56639357474563904c28e30f9d4d9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9776911d31b4438397a74f745291cafe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yOr6KErj66Fk"
      },
      "source": [
        "Based on the HuggingFace CLM example https://github.com/huggingface/transformers/blob/master/examples/pytorch/language-modeling/run_clm.py "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aC9kRemVYakz"
      },
      "source": [
        "# Initial Transformers prereqs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JjIHax6y10s_"
      },
      "source": [
        "%%capture\n",
        "! pip install transformers datasets"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-0BIFZsg7E9S"
      },
      "source": [
        "import logging\n",
        "import math\n",
        "import os\n",
        "import sys\n",
        "from dataclasses import dataclass, field\n",
        "from typing import Optional\n",
        "\n",
        "import transformers\n",
        "from transformers import (\n",
        "    CONFIG_MAPPING,\n",
        "    MODEL_FOR_CAUSAL_LM_MAPPING,\n",
        "    AutoConfig,\n",
        "    AutoModelForCausalLM,\n",
        "    AutoTokenizer,\n",
        "    HfArgumentParser,\n",
        "    Trainer,\n",
        "    TrainingArguments,\n",
        "    default_data_collator,\n",
        "    set_seed,\n",
        ")\n",
        "from datasets import load_dataset\n",
        "from transformers.testing_utils import CaptureLogger\n",
        "from transformers.trainer_utils import get_last_checkpoint\n",
        "from transformers.utils import check_min_version\n",
        "from transformers.utils.versions import require_version"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "osjTRoSU7GDG"
      },
      "source": [
        "logger = logging.getLogger(__name__)\n",
        "\n",
        "\n",
        "MODEL_CONFIG_CLASSES = list(MODEL_FOR_CAUSAL_LM_MAPPING.keys())\n",
        "MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)\n",
        "\n",
        "\n",
        "@dataclass\n",
        "class ModelArguments:\n",
        "    \"\"\"\n",
        "    Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.\n",
        "    \"\"\"\n",
        "\n",
        "    model_name_or_path: Optional[str] = field(\n",
        "        default=None,\n",
        "        metadata={\n",
        "            \"help\": \"The model checkpoint for weights initialization.\"\n",
        "            \"Don't set if you want to train a model from scratch.\"\n",
        "        },\n",
        "    )\n",
        "    model_type: Optional[str] = field(\n",
        "        default=None,\n",
        "        metadata={\"help\": \"If training from scratch, pass a model type from the list: \" + \", \".join(MODEL_TYPES)},\n",
        "    )\n",
        "    config_overrides: Optional[str] = field(\n",
        "        default=None,\n",
        "        metadata={\n",
        "            \"help\": \"Override some existing default config settings when a model is trained from scratch. Example: \"\n",
        "            \"n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index\"\n",
        "        },\n",
        "    )\n",
        "    config_name: Optional[str] = field(\n",
        "        default=None, metadata={\"help\": \"Pretrained config name or path if not the same as model_name\"}\n",
        "    )\n",
        "    tokenizer_name: Optional[str] = field(\n",
        "        default=None, metadata={\"help\": \"Pretrained tokenizer name or path if not the same as model_name\"}\n",
        "    )\n",
        "    cache_dir: Optional[str] = field(\n",
        "        default=None,\n",
        "        metadata={\"help\": \"Where do you want to store the pretrained models downloaded from huggingface.co\"},\n",
        "    )\n",
        "    use_fast_tokenizer: bool = field(\n",
        "        default=True,\n",
        "        metadata={\"help\": \"Whether to use one of the fast tokenizer (backed by the tokenizers library) or not.\"},\n",
        "    )\n",
        "    model_revision: str = field(\n",
        "        default=\"main\",\n",
        "        metadata={\"help\": \"The specific model version to use (can be a branch name, tag name or commit id).\"},\n",
        "    )\n",
        "    use_auth_token: bool = field(\n",
        "        default=False,\n",
        "        metadata={\n",
        "            \"help\": \"Will use the token generated when running `transformers-cli login` (necessary to use this script \"\n",
        "            \"with private models).\"\n",
        "        },\n",
        "    )\n",
        "\n",
        "    def __post_init__(self):\n",
        "        if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None):\n",
        "            raise ValueError(\n",
        "                \"--config_overrides can't be used in combination with --config_name or --model_name_or_path\"\n",
        "            )"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dJ-vzQye7lXd"
      },
      "source": [
        "training_args = TrainingArguments(\n",
        "    output_dir=\"./output\",\n",
        "    do_train=True,\n",
        "    overwrite_output_dir=True,\n",
        "    per_device_train_batch_size=1,\n",
        "    save_strategy=\"no\"\n",
        ")"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qlkpEirF8GSP"
      },
      "source": [
        "model_args = ModelArguments(\"gpt2\", \"gpt\")"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c6bscHVm9ZDe",
        "outputId": "55b91629-f5fa-4fe5-f104-1d4bc71f836f"
      },
      "source": [
        "! wget https://raw.githubusercontent.com/MonsoonNLP/gpt-nyc/main/combined.csv\n",
        "\n",
        "import csv\n",
        "op = open('./csvout.csv', 'w')\n",
        "wrt = csv.writer(op)\n",
        "\n",
        "with open('./combined.csv', 'r') as srcfile:\n",
        "  rdr = csv.reader(srcfile)\n",
        "  for row in rdr:\n",
        "    wrt.writerow(['. '.join(row), '1'])"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2021-11-15 23:35:35--  https://raw.githubusercontent.com/MonsoonNLP/gpt-nyc/main/combined.csv\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.111.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 9675046 (9.2M) [text/plain]\n",
            "Saving to: ‘combined.csv’\n",
            "\n",
            "combined.csv        100%[===================>]   9.23M  --.-KB/s    in 0.1s    \n",
            "\n",
            "2021-11-15 23:35:36 (83.8 MB/s) - ‘combined.csv’ saved [9675046/9675046]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 186,
          "referenced_widgets": [
            "183fd8b8da7a420da59475474e826c7f",
            "d3a61fa2a9c0437f8e5786bafafc97e9",
            "1fe3733155be4201b4a631628e6f387c",
            "e89139d5d03b4502ae5e16a4891fb1f2",
            "2db449311da94c3fb53d774e9a0ee91a",
            "b1936d69238f4534a77ebc613c496daa",
            "edcd1d485a2f42bcac26805b8e7fca57",
            "929c39ea765e45e8a88e460e6e21128f",
            "22c014b4f7bf4cbda3a91b7f086ba59c",
            "9ee60a02ee044e59bc53753477cd359b",
            "03352d80fc48434ebc26052e16cb16e4",
            "618ddd39e1f34711b7c8868f76d1b763",
            "349b3234095643d486d13e659a00d136",
            "b4d9902c8c2147ba861183e1e50c46f5",
            "50944cda66f14ab78d0a15262cfc1bd1",
            "1eef083e7e8e4247b4cb6be0dd64a570",
            "fdfd03df54304f10a11edb6ca840bb67",
            "0be3581e068244ffac134abbebd903ca",
            "a2c1d424ad29476a913bf4f4868b7b2b",
            "8de6c28c128e4ebc87ba171ff625895f",
            "48f5fa53d94f4ff9a10a5ca626ce2396",
            "5259a1826ff34a5185968dcf073bc205",
            "ed39797f16b841809a8e55f2eed8f3e6",
            "3a25defe466d4a0fb31110a04f4ea46e",
            "9f39466a86134e63bfa8fe3690dd4bc0",
            "fb1416c8cbc747eea07bd5319e4b53e7",
            "ea55a4bb0a1f442a90b83fb1dc1a97e9",
            "d8bd1b83ee01468d98fbea5dedef0639",
            "8dbd18575f1a4090bbaab75bd9cc710b",
            "8a12ae8fade549008d0f9a125a4d4b56",
            "861a6feb84a14864b841a5d9322e668a",
            "31b8aad9d82c4147b2c20834de59fbfa",
            "2814618811de4f23863d571292ac3039"
          ]
        },
        "id": "a8rKVJTBNjw-",
        "outputId": "8fa76c1a-aa93-416d-dae7-eb15795ae913"
      },
      "source": [
        "raw_datasets = {}\n",
        "raw_datasets = load_dataset(\"csv\", data_files=\"./csvout.csv\", column_names=['text', 'extra'])"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Using custom data configuration default-fcd7d1bf2102ce0c\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset csv/default to /root/.cache/huggingface/datasets/csv/default-fcd7d1bf2102ce0c/0.0.0/bf68a4c4aefa545d0712b2fcbb1b327f905bbe2f6425fbc5e8c25234acb9e14a...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "183fd8b8da7a420da59475474e826c7f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "618ddd39e1f34711b7c8868f76d1b763",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset csv downloaded and prepared to /root/.cache/huggingface/datasets/csv/default-fcd7d1bf2102ce0c/0.0.0/bf68a4c4aefa545d0712b2fcbb1b327f905bbe2f6425fbc5e8c25234acb9e14a. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ed39797f16b841809a8e55f2eed8f3e6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9riq9IAgOB8Y",
        "outputId": "0ec79483-6a5f-465e-ed3a-df8e55bca8da"
      },
      "source": [
        "raw_datasets[\"train\"].column_names"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['text', 'extra']"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HvUDtHRSlPWK"
      },
      "source": [
        "# Detecting last checkpoint.\n",
        "last_checkpoint = None\n",
        "if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:\n",
        "    last_checkpoint = get_last_checkpoint(training_args.output_dir)\n",
        "    if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:\n",
        "        raise ValueError(\n",
        "            f\"Output directory ({training_args.output_dir}) already exists and is not empty. \"\n",
        "            \"Use --overwrite_output_dir to overcome.\"\n",
        "        )\n",
        "    elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:\n",
        "        logger.info(\n",
        "            f\"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change \"\n",
        "            \"the `--output_dir` or add `--overwrite_output_dir` to train from scratch.\"\n",
        "        )\n",
        "\n",
        "# Set seed before initializing model.\n",
        "set_seed(training_args.seed)\n",
        "\n",
        "config_kwargs = {\n",
        "    \"cache_dir\": model_args.cache_dir,\n",
        "    \"revision\": model_args.model_revision,\n",
        "    \"use_auth_token\": True if model_args.use_auth_token else None,\n",
        "}"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xWGPi6Vnn2p3"
      },
      "source": [
        "# Track a single true-token match in GPT-2 over AskNYC Qs dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 388
        },
        "id": "_-BHSu6I7Olr",
        "outputId": "7a8f4da6-ae58-4323-a818-fb2ad6485919"
      },
      "source": [
        "if model_args.config_name:\n",
        "    config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs)\n",
        "elif model_args.model_name_or_path:\n",
        "    config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)\n",
        "else:\n",
        "    config = CONFIG_MAPPING[model_args.model_type]()\n",
        "    logger.warning(\"You are instantiating a new config instance from scratch.\")\n",
        "    if model_args.config_overrides is not None:\n",
        "        logger.info(f\"Overriding config: {model_args.config_overrides}\")\n",
        "        config.update_from_string(model_args.config_overrides)\n",
        "\n",
        "tokenizer_kwargs = {\n",
        "    \"cache_dir\": model_args.cache_dir,\n",
        "    \"use_fast\": model_args.use_fast_tokenizer,\n",
        "    \"revision\": model_args.model_revision,\n",
        "    \"use_auth_token\": True if model_args.use_auth_token else None,\n",
        "}\n",
        "if model_args.tokenizer_name:\n",
        "    tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs)\n",
        "elif model_args.model_name_or_path:\n",
        "    tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, **tokenizer_kwargs)\n",
        "else:\n",
        "    raise ValueError(\n",
        "        \"You are instantiating a new tokenizer from scratch. This is not supported by this script.\"\n",
        "        \"You can do it from another script, save it, and load it from here, using --tokenizer_name.\"\n",
        "    )\n",
        "\n",
        "if model_args.model_name_or_path:\n",
        "    model = AutoModelForCausalLM.from_pretrained(\n",
        "        model_args.model_name_or_path,\n",
        "        from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n",
        "        config=config,\n",
        "        cache_dir=model_args.cache_dir,\n",
        "        revision=model_args.model_revision,\n",
        "        use_auth_token=True if model_args.use_auth_token else None,\n",
        "    )\n",
        "else:\n",
        "    model = AutoModelForCausalLM.from_config(config)\n",
        "    n_params = sum(dict((p.data_ptr(), p.numel()) for p in model.parameters()).values())\n",
        "    logger.info(f\"Training new model from scratch - Total size={n_params/2**20:.2f}M params\")\n",
        "\n",
        "model.resize_token_embeddings(len(tokenizer))\n",
        "\n",
        "# Preprocessing the datasets.\n",
        "# First we tokenize all the texts.\n",
        "if training_args.do_train:\n",
        "    column_names = raw_datasets[\"train\"].column_names\n",
        "else:\n",
        "    column_names = raw_datasets[\"validation\"].column_names\n",
        "text_column_name = \"text\" if \"text\" in column_names else column_names[0]\n",
        "\n",
        "# since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function\n",
        "tok_logger = transformers.utils.logging.get_logger(\"transformers.tokenization_utils_base\")\n",
        "\n",
        "def tokenize_function(examples):\n",
        "    with CaptureLogger(tok_logger) as cl:\n",
        "        output = tokenizer(examples[text_column_name])\n",
        "    # clm input could be much much longer than block_size\n",
        "    if \"Token indices sequence length is longer than the\" in cl.out:\n",
        "        tok_logger.warning(\n",
        "            \"^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\"\n",
        "        )\n",
        "    return output\n",
        "\n",
        "with training_args.main_process_first(desc=\"dataset map tokenization\"):\n",
        "    tokenized_datasets = raw_datasets.map(\n",
        "        tokenize_function,\n",
        "        batched=True,\n",
        "        remove_columns=column_names,\n",
        "        desc=\"Running tokenizer on dataset\",\n",
        "    )\n",
        "\n",
        "# Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size.\n",
        "def group_texts(examples):\n",
        "    # Concatenate all texts.\n",
        "    concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}\n",
        "    total_length = len(concatenated_examples[list(examples.keys())[0]])\n",
        "    # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can\n",
        "    # customize this part to your needs.\n",
        "    if total_length >= tokenizer.model_max_length:\n",
        "        total_length = (total_length // tokenizer.model_max_length) * tokenizer.model_max_length\n",
        "    # Split by chunks of max_len.\n",
        "    result = {\n",
        "        k: [t[i : i + tokenizer.model_max_length] for i in range(0, total_length, tokenizer.model_max_length)]\n",
        "        for k, t in concatenated_examples.items()\n",
        "    }\n",
        "    result[\"labels\"] = result[\"input_ids\"].copy()\n",
        "    return result\n",
        "\n",
        "# Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a remainder\n",
        "# for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value might be slower\n",
        "# to preprocess.\n",
        "#\n",
        "# To speed up this part, we use multiprocessing. See the documentation of the map method for more information:\n",
        "# https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map\n",
        "\n",
        "with training_args.main_process_first(desc=\"grouping texts together\"):\n",
        "    lm_datasets = tokenized_datasets.map(\n",
        "        group_texts,\n",
        "        batched=True,\n",
        "        desc=f\"Grouping texts in chunks of {tokenizer.model_max_length}\",\n",
        "    )\n",
        "\n",
        "if training_args.do_train:\n",
        "    if \"train\" not in tokenized_datasets:\n",
        "        raise ValueError(\"--do_train requires a train dataset\")\n",
        "    train_dataset = lm_datasets[\"train\"]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-15-01cb1e5ba32e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0mcache_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel_args\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcache_dir\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m         \u001b[0mrevision\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel_args\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_revision\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m         \u001b[0muse_auth_token\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmodel_args\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_auth_token\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     36\u001b[0m     )\n\u001b[1;32m     37\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/transformers/models/auto/auto_factory.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m    417\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model_mapping\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    418\u001b[0m             \u001b[0mmodel_class\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_get_model_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model_mapping\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 419\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mmodel_class\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpretrained_model_name_or_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmodel_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    420\u001b[0m         raise ValueError(\n\u001b[1;32m    421\u001b[0m             \u001b[0;34mf\"Unrecognized configuration class {config.__class__} for this kind of AutoModel: {cls.__name__}.\\n\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m   1400\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1401\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mno_init_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_enable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_fast_init\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1402\u001b[0;31m                 \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmodel_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mmodel_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1403\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1404\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mfrom_pt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/transformers/models/gpt2/modeling_gpt2.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, config)\u001b[0m\n\u001b[1;32m    945\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    946\u001b[0m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 947\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mGPT2Model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    948\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlm_head\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_embd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/transformers/models/gpt2/modeling_gpt2.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, config)\u001b[0m\n\u001b[1;32m    675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    676\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membd_pdrop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 677\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModuleList\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mGPT2Block\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer_idx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_hidden_layers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    678\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mln_f\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLayerNorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membed_dim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer_norm_epsilon\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    679\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/transformers/models/gpt2/modeling_gpt2.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    676\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membd_pdrop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 677\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModuleList\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mGPT2Block\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer_idx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_hidden_layers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    678\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mln_f\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLayerNorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membed_dim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer_norm_epsilon\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    679\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/transformers/models/gpt2/modeling_gpt2.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, config, layer_idx)\u001b[0m\n\u001b[1;32m    371\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    372\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mln_1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLayerNorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer_norm_epsilon\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 373\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mGPT2Attention\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer_idx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlayer_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    374\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mln_2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLayerNorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer_norm_epsilon\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    375\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/transformers/models/gpt2/modeling_gpt2.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, config, is_cross_attention, layer_idx)\u001b[0m\n\u001b[1;32m    140\u001b[0m         self.register_buffer(\n\u001b[1;32m    141\u001b[0m             \u001b[0;34m\"bias\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 142\u001b[0;31m             torch.tril(torch.ones((max_positions, max_positions), dtype=torch.uint8)).view(\n\u001b[0m\u001b[1;32m    143\u001b[0m                 \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_positions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_positions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m             ),\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "dB1_4ItwH1Pz",
        "outputId": "1f28e7a8-95be-49cc-9f50-e1af1bec7de5"
      },
      "source": [
        "# this is a token I decided to measure loss on in the Croatian dataset\n",
        "# written it is \"aj\" (suffix token)\n",
        "MONITOR_TOKEN_ID = 1228\n",
        "tokenizer.decode([MONITOR_TOKEN_ID])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'aj'"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wdH0ik6bOYJD",
        "outputId": "bda2a6ea-08b1-4946-db89-e50b2b1db3dd"
      },
      "source": [
        "# this is the token I decided to monitor in GPT-NYC dataset\n",
        "print(tokenizer([' train']))\n",
        "MONITOR_TOKEN_ID = 4512"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': [[4512]], 'attention_mask': [[1]]}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8CUia_cFYosD"
      },
      "source": [
        "## Overriding Trainer.compute_loss to log"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YhWe7JD688qR"
      },
      "source": [
        "# subclass of Trainer https://huggingface.co/transformers/main_classes/trainer.html\n",
        "class TokenTrackingTrainer(Trainer):\n",
        "  def compute_loss(self, model, inputs, return_outputs=False):\n",
        "    #print(inputs['input_ids'])\n",
        "    outputs = model(**inputs)['logits']\n",
        "    for batch_num, batch_output in enumerate(outputs):\n",
        "      for token_num, token_output in enumerate(batch_output):\n",
        "        correct_token = inputs['input_ids'][batch_num][token_num]\n",
        "        #print(\"correct token:\")\n",
        "        #print(correct_token)\n",
        "        if correct_token.item() == MONITOR_TOKEN_ID:\n",
        "          #print(\"predictions:\")\n",
        "          #print(token_output)\n",
        "          if correct_token < len(token_output):\n",
        "            #print(\"local prob:\")\n",
        "            print(token_output[inputs['input_ids'][batch_num][token_num]].item())\n",
        "\n",
        "    return super().compute_loss(model, inputs, return_outputs)\n",
        "\n",
        "trainer = TokenTrackingTrainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=train_dataset if training_args.do_train else None,\n",
        "    eval_dataset=eval_dataset if training_args.do_eval else None,\n",
        "    tokenizer=tokenizer,\n",
        "    # Data collator will default to DataCollatorWithPadding, so we change it.\n",
        "    data_collator=default_data_collator,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "id": "eE5OXNfgCDj4",
        "outputId": "760442ec-17b2-4205-d38c-2c82e9ae5cd6"
      },
      "source": [
        "# this attempt to track tokens from a callback was flawed because the callback does not see inputs\n",
        "\"\"\"\n",
        "from transformers import TrainerCallback, TrainerState, TrainerControl\n",
        "\n",
        "class TokenLossCallback(TrainerCallback):\n",
        "  #def on_substep_end(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, **kwargs):\n",
        "  #  print(\"substep\")\n",
        "  \n",
        "  def on_step_end(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, **kwargs):\n",
        "    print(\"step\")\n",
        "    print(kwargs.keys())\n",
        "\n",
        "trainer.add_callback(TokenLossCallback())\n",
        "\"\"\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'\\nfrom transformers import TrainerCallback, TrainerState, TrainerControl\\n\\nclass TokenLossCallback(TrainerCallback):\\n  #def on_substep_end(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, **kwargs):\\n  #  print(\"substep\")\\n  \\n  def on_step_end(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, **kwargs):\\n    print(\"step\")\\n    print(kwargs.keys())\\n\\ntrainer.add_callback(TokenLossCallback())\\n'"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ru9MRfAOYwnD"
      },
      "source": [
        "## Track loss of model and individual token during training\n",
        "\n",
        "I cleared the logs on these during training, but it started all -150 to -60, by 0.28 epochs we had some positive scores :)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AbALcWPr8ZR2"
      },
      "source": [
        "checkpoint = None\n",
        "if training_args.resume_from_checkpoint is not None:\n",
        "    checkpoint = training_args.resume_from_checkpoint\n",
        "elif last_checkpoint is not None:\n",
        "    checkpoint = last_checkpoint\n",
        "train_result = trainer.train(resume_from_checkpoint=checkpoint)\n",
        "trainer.save_model()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6leDmfQtY9fF"
      },
      "source": [
        "# Add NYC tokens to GPT-2 and track them all during fine-tuning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A7Od7YlTi5vh"
      },
      "source": [
        "## Tokenizer / model setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 176,
          "referenced_widgets": [
            "159905c697814dd9b457c71096afd1f6",
            "e63469e16e1b463c8ff5bcad1a4657c6",
            "3139f932c33e415780b248d14c1ff51c",
            "68eaa842c14c4d69aa8a7b9dc3908769",
            "38f876fef866469aac6ca4769b0206f2",
            "1da3cde112914ce190ac3c802bb97f56",
            "28ccbfaa32194b2a9d55ee4e63ab510b",
            "e04d1cd1e6494b5eaba83008c472ff93",
            "1186b6992b3640ebac05e49ab7291efa",
            "73d8aae92f0b4fa185cbc61d834d2a3c",
            "294556f9af424526a0e3760677af8715",
            "91719f65f2354310aeb45a5b06a28c6f",
            "af071bc680da46c1a6038f0d6515bb86",
            "cfe4732816364b6da651d54ca131c972",
            "5836da7037234081b106940dd083083c",
            "0e036abea87f49afab4b60316914777e",
            "ee4fd5d947324f66a1109f1d084ec510",
            "9d995c8ad1c3418eb46db63771ecb68d",
            "40ff380dfc994589ba78c75a0cecba74",
            "d5c9f5d1f4214bfeaf6e715dc9dd635a",
            "569c40ac513249b9b8580d99fc36c817",
            "8b174b0125dd485482335d7a732b45d6",
            "8c6c7032705947c0b059fc1643e618df",
            "6afbca38b20e4af9bb93d9ac82924e5a",
            "82bb714db3ed496980c2dff1ff304ffd",
            "4413b6b8764045c6892038fe421da18a",
            "14b1a59b71754471ae8114a3f5b29998",
            "7b0dcbaefeed4c03a39bc70cd22ea634",
            "a77f9e75aa5f47e1b06e66ac9d46daad",
            "12fad5558d68406b8b24ed48e573d6d6",
            "f0fe9a5e5a4d4a05ba71ad4675343af3",
            "f7387ccb857f44b2a71a348d09e99d13",
            "d32e5c0b242c4975982975978dd8b386",
            "b3cc9a6582444ed1b230ebe62903899d",
            "773e6f8aa2f64c738268561a9058ed85",
            "f53bd7d4842d40aeaa370fa4ead86147",
            "18bf96a80c10445a8d0568e252951e30",
            "ca88243771524d1c991f004a157c3a95",
            "b030e6a1b007481a8c37bbe4e4275fbf",
            "68b50b59da1844d1863abb82f2ac5331",
            "33403267bff2447f84bbbda19f2915c3",
            "0f4334b894b845cd8454bd673a9b986d",
            "ee4e0206133a4351bbf2b922c5346dad",
            "6afb1a29518f472b963d1c65be76b54f",
            "b96a1bcebf054f0581c8ccd4caa80127",
            "758d09e5883c473e8bbdc6c9ec63cc33",
            "d5d3bcb4c20e48a9b8a42d808ee982df",
            "0b00094ede694737b3d381c3f3405b4a",
            "c74c64bcf8fa42f2a61f5ea7c6f87be3",
            "428ba775d3584cf786d5d2f3700c6e61",
            "79a540007d264607beb4db016b7faa67",
            "cde043d7cc414dba981e66353ddad925",
            "7e26f9b3f6f34505bbec766f028a0bb5",
            "feddff02f44446538cd2625060a558f0",
            "eeaf0b8dd30a4aee9cc2620138cf79e9"
          ]
        },
        "id": "9Q5GXCkUic3Y",
        "outputId": "c7513d00-35c0-4f96-cb83-4986c749a644"
      },
      "source": [
        "# load original tokenizer / model\n",
        "from transformers import AutoTokenizer, AutoModel\n",
        "tokenizer = AutoTokenizer.from_pretrained('gpt2')\n",
        "model = AutoModel.from_pretrained('gpt2')"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "159905c697814dd9b457c71096afd1f6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/665 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "91719f65f2354310aeb45a5b06a28c6f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/0.99M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8c6c7032705947c0b059fc1643e618df",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/446k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b3cc9a6582444ed1b230ebe62903899d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.29M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b96a1bcebf054f0581c8ccd4caa80127",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/523M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pZlEaNVBj46V",
        "outputId": "5f8934fb-52a0-4977-eef1-80ac3b592272"
      },
      "source": [
        "# original token length\n",
        "len(tokenizer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "50257"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9d4-8smKihmO",
        "outputId": "f7627278-9463-4024-f6c1-21ee70d1539b"
      },
      "source": [
        "! wget https://raw.githubusercontent.com/MonsoonNLP/gpt-nyc/main/newnyc.txt\n",
        "\n",
        "def unique(list1):\n",
        "    unique_list = []     \n",
        "    for x in list1:\n",
        "        if x not in unique_list:\n",
        "            unique_list.append(x)\n",
        "    return unique_list\n",
        "\n",
        "with open('newnyc.txt', 'r') as vfile:\n",
        "  new_nyc = unique(vfile.read().strip().split(\"\\n\"))\n",
        "new_nyc[:10]"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2021-11-15 23:36:12--  https://raw.githubusercontent.com/MonsoonNLP/gpt-nyc/main/newnyc.txt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.109.133, 185.199.111.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 7204 (7.0K) [text/plain]\n",
            "Saving to: ‘newnyc.txt’\n",
            "\n",
            "newnyc.txt          100%[===================>]   7.04K  --.-KB/s    in 0s      \n",
            "\n",
            "2021-11-15 23:36:12 (63.5 MB/s) - ‘newnyc.txt’ saved [7204/7204]\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['nyc',\n",
              " 'Astoria',\n",
              " 'Williamsburg',\n",
              " 'midtown',\n",
              " 'Bushwick',\n",
              " 'UWS',\n",
              " 'UES',\n",
              " 'yelp',\n",
              " 'manhattan',\n",
              " 'Midtown']"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6Ho0k9QBixW_",
        "outputId": "3b6a3943-df78-4dd1-8551-c1b7210dcbe3"
      },
      "source": [
        "tokenizer.add_tokens(new_nyc)\n",
        "tokenizer.add_special_tokens({'pad_token': '[PAD]'})\n",
        "model.resize_token_embeddings(len(tokenizer))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Embedding(51004, 768)"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "971Li2XQiz1E"
      },
      "source": [
        "tokenizer.save_pretrained(\"gpt-nyc\")\n",
        "model.save_pretrained(\"gpt-nyc\")\n",
        "del tokenizer\n",
        "del model"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "92RmBpbBoqd4"
      },
      "source": [
        "### If you already created GPT-NYC tokens model you can jump to here"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gaWsPLN6jJS_"
      },
      "source": [
        "model_args = ModelArguments(\"./gpt-nyc\", \"gpt\")"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136,
          "referenced_widgets": [
            "bb1881cc120749978cf226fa908e1de2",
            "a65003e7d3f24c48b6ced56f41631390",
            "c76e5749cc9c49738e669bd7130ba92c",
            "1685b2a6b7f54fab94ea6a506b3ba211",
            "d2fe9ed993fa4f11bad266c9b7242ec5",
            "bd4ea502336c4d8db0cb0a6675f69a91",
            "925b4da85dc34d1585540c822fd26474",
            "cf568ab2a00944348d22b852744ead75",
            "a49288faa99846c3bace7aba46ba7361",
            "1e7b9b87512648b5838dd5a9d5519d87",
            "1840298801444267bc3f11e1b05cdcb6",
            "55b5a90e0e1941898916e20aca4ed5e8",
            "dd0c646708164d89936106f437eec677",
            "3538feac33214a4d94e19bf33d64811c",
            "6250a0ed184a4c86a82f66ffc72aea2e",
            "fdffbbf4a50a4dcd9105405040cb9057",
            "6df45c6fef00459aa36b14beaf0a5c6b",
            "6937a423b8cc42538ef771d4270e5eb5",
            "58eadf9845124173815a0684f356e9ba",
            "8804c0b5e6684f9da4d41c7a3c57404f",
            "5e56639357474563904c28e30f9d4d9b",
            "9776911d31b4438397a74f745291cafe"
          ]
        },
        "id": "U3qn7bAdi76R",
        "outputId": "ae5e4b97-7e22-4b48-fd73-2e0a91a02899"
      },
      "source": [
        "if model_args.config_name:\n",
        "    config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs)\n",
        "elif model_args.model_name_or_path:\n",
        "    config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)\n",
        "else:\n",
        "    config = CONFIG_MAPPING[model_args.model_type]()\n",
        "    logger.warning(\"You are instantiating a new config instance from scratch.\")\n",
        "    if model_args.config_overrides is not None:\n",
        "        logger.info(f\"Overriding config: {model_args.config_overrides}\")\n",
        "        config.update_from_string(model_args.config_overrides)\n",
        "\n",
        "tokenizer_kwargs = {\n",
        "    \"cache_dir\": model_args.cache_dir,\n",
        "    \"use_fast\": model_args.use_fast_tokenizer,\n",
        "    \"revision\": model_args.model_revision,\n",
        "    \"use_auth_token\": True if model_args.use_auth_token else None,\n",
        "}\n",
        "if model_args.tokenizer_name:\n",
        "    tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs)\n",
        "elif model_args.model_name_or_path:\n",
        "    tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, **tokenizer_kwargs)\n",
        "else:\n",
        "    raise ValueError(\n",
        "        \"You are instantiating a new tokenizer from scratch. This is not supported by this script.\"\n",
        "        \"You can do it from another script, save it, and load it from here, using --tokenizer_name.\"\n",
        "    )\n",
        "\n",
        "if model_args.model_name_or_path:\n",
        "    model = AutoModelForCausalLM.from_pretrained(\n",
        "        model_args.model_name_or_path,\n",
        "        from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n",
        "        config=config,\n",
        "        cache_dir=model_args.cache_dir,\n",
        "        revision=model_args.model_revision,\n",
        "        use_auth_token=True if model_args.use_auth_token else None,\n",
        "    )\n",
        "else:\n",
        "    model = AutoModelForCausalLM.from_config(config)\n",
        "    n_params = sum(dict((p.data_ptr(), p.numel()) for p in model.parameters()).values())\n",
        "    logger.info(f\"Training new model from scratch - Total size={n_params/2**20:.2f}M params\")\n",
        "\n",
        "model.resize_token_embeddings(len(tokenizer))\n",
        "\n",
        "# Preprocessing the datasets.\n",
        "# First we tokenize all the texts.\n",
        "if training_args.do_train:\n",
        "    column_names = raw_datasets[\"train\"].column_names\n",
        "else:\n",
        "    column_names = raw_datasets[\"validation\"].column_names\n",
        "text_column_name = \"text\" if \"text\" in column_names else column_names[0]\n",
        "\n",
        "# since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function\n",
        "tok_logger = transformers.utils.logging.get_logger(\"transformers.tokenization_utils_base\")\n",
        "\n",
        "def tokenize_function(examples):\n",
        "    with CaptureLogger(tok_logger) as cl:\n",
        "        output = tokenizer(examples[text_column_name])\n",
        "    # clm input could be much much longer than block_size\n",
        "    if \"Token indices sequence length is longer than the\" in cl.out:\n",
        "        tok_logger.warning(\n",
        "            \"^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\"\n",
        "        )\n",
        "    return output\n",
        "\n",
        "with training_args.main_process_first(desc=\"dataset map tokenization\"):\n",
        "    tokenized_datasets = raw_datasets.map(\n",
        "        tokenize_function,\n",
        "        batched=True,\n",
        "        remove_columns=column_names,\n",
        "        desc=\"Running tokenizer on dataset\",\n",
        "    )\n",
        "\n",
        "def group_texts(examples):\n",
        "    concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}\n",
        "    total_length = len(concatenated_examples[list(examples.keys())[0]])\n",
        "    if total_length >= tokenizer.model_max_length:\n",
        "        total_length = (total_length // tokenizer.model_max_length) * tokenizer.model_max_length\n",
        "    result = {\n",
        "        k: [t[i : i + tokenizer.model_max_length] for i in range(0, total_length, tokenizer.model_max_length)]\n",
        "        for k, t in concatenated_examples.items()\n",
        "    }\n",
        "    result[\"labels\"] = result[\"input_ids\"].copy()\n",
        "    return result\n",
        "\n",
        "with training_args.main_process_first(desc=\"grouping texts together\"):\n",
        "    lm_datasets = tokenized_datasets.map(\n",
        "        group_texts,\n",
        "        batched=True,\n",
        "        desc=f\"Grouping texts in chunks of {tokenizer.model_max_length}\",\n",
        "    )\n",
        "\n",
        "if training_args.do_train:\n",
        "    if \"train\" not in tokenized_datasets:\n",
        "        raise ValueError(\"--do_train requires a train dataset\")\n",
        "    train_dataset = lm_datasets[\"train\"]"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bb1881cc120749978cf226fa908e1de2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Running tokenizer on dataset:   0%|          | 0/14 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (1229 > 1024). Running this sequence through the model will result in indexing errors\n",
            "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "55b5a90e0e1941898916e20aca4ed5e8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Grouping texts in chunks of 1024:   0%|          | 0/14 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7ez2rDVsj-vy",
        "outputId": "08e5562c-e23c-449a-9740-571dbdee815f"
      },
      "source": [
        "print(tokenizer.decode([50257]))\n",
        "NEW_TOKEN_START = 50257"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "nyc\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bUPiW5Nbx17O"
      },
      "source": [
        "## Tracking only true matches at first"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WvuAYVb9jlvt"
      },
      "source": [
        "# subclass of Trainer https://huggingface.co/transformers/main_classes/trainer.html\n",
        "class TokensTrackingTrainer(Trainer):\n",
        "  def compute_loss(self, model, inputs, return_outputs=False):\n",
        "    #print(inputs['input_ids'])\n",
        "    outputs = model(**inputs)['logits']\n",
        "    avg_fit = 0\n",
        "    fit_tokens = 0\n",
        "    for batch_num, batch_output in enumerate(outputs):\n",
        "      for token_num, token_output in enumerate(batch_output):\n",
        "        correct_token = inputs['input_ids'][batch_num][token_num]\n",
        "        #print(\"correct token:\")\n",
        "        #print(correct_token)\n",
        "        if correct_token.item() >= NEW_TOKEN_START:\n",
        "          #print(\"predictions:\")\n",
        "          #print(token_output)\n",
        "          if correct_token < len(token_output):\n",
        "            individual_fit = token_output[inputs['input_ids'][batch_num][token_num]].item()\n",
        "            avg_fit += individual_fit\n",
        "            fit_tokens += 1\n",
        "    if fit_tokens > 0:\n",
        "      print(avg_fit / fit_tokens)\n",
        "\n",
        "    return super().compute_loss(model, inputs, return_outputs)\n",
        "\n",
        "trainer = TokensTrackingTrainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=train_dataset if training_args.do_train else None,\n",
        "    eval_dataset=eval_dataset if training_args.do_eval else None,\n",
        "    tokenizer=tokenizer,\n",
        "    data_collator=default_data_collator,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EQtFPM6pkkhX"
      },
      "source": [
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QXCje_XeaqQ7"
      },
      "source": [
        "## Loss calculation (i.e. true/false x positives/negatives)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L10YtFcj5Rr2",
        "outputId": "ad77fc61-2d51-4d8b-e741-655d97252569"
      },
      "source": [
        "tokenizer([' bagel', ' bagels', ' Sunnyside', ' DUMBO'])"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [[220, 50289], [220, 50278], [220, 50299], [220, 50326]], 'attention_mask': [[1, 1], [1, 1], [1, 1], [1, 1]]}"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-fk_AGgW5V80"
      },
      "source": [
        "TRACK_TOKENS = [50278, 50289]"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lgpQRRVVw7yn"
      },
      "source": [
        "# loss function chosen based on GPT2 implementation https://github.com/huggingface/transformers/blob/391db836ab7ed2ca61c51a7cf1b135b6ab92be58/transformers/modeling_gpt2.py#L539\n",
        "from torch.nn import CrossEntropyLoss\n",
        "loss_appearances = []\n",
        "op = []"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GY9sn8havtfF"
      },
      "source": [
        "class TokensLossTrainer(Trainer):\n",
        "  def compute_loss(self, model, inputs, return_outputs=False):\n",
        "    outputs = model(**inputs)\n",
        "\n",
        "    out_logits = outputs['logits']\n",
        "    out_labels = inputs['input_ids']\n",
        "    focus_tokens = [[], []]\n",
        "    match_tokens = 0\n",
        "\n",
        "    for batch_num, batch_output in enumerate(outputs['logits']):\n",
        "        for token_num, token_output in enumerate(batch_output):\n",
        "          correct_token = inputs['input_ids'][batch_num][token_num]\n",
        "          #if correct_token.item() >= NEW_TOKEN_START:\n",
        "          if correct_token.item() in TRACK_TOKENS:\n",
        "            focus_tokens[0].append(batch_num)\n",
        "            focus_tokens[1].append(token_num)\n",
        "            match_tokens += 1\n",
        "    \n",
        "    op.append(out_logits)\n",
        "    shift_logits = out_logits[focus_tokens].contiguous()\n",
        "    shift_labels = out_labels[focus_tokens].contiguous()\n",
        "    \n",
        "    loss_fct = CrossEntropyLoss(ignore_index=-1)\n",
        "    loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)),\n",
        "                    shift_labels.view(-1))\n",
        "    if match_tokens > 0:\n",
        "      loss = loss.item()\n",
        "      loss_appearances.append([outputs['loss'].item(), loss])\n",
        "      print(loss)\n",
        "\n",
        "    return super().compute_loss(model, inputs, return_outputs)\n",
        "\n",
        "trainer = TokensLossTrainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=train_dataset if training_args.do_train else None,\n",
        "    eval_dataset=eval_dataset if training_args.do_eval else None,\n",
        "    tokenizer=tokenizer,\n",
        "    data_collator=default_data_collator,\n",
        ")"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r5Fb8j1IxMmB"
      },
      "source": [
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dC23vPFhkQbn",
        "outputId": "1b65ea25-282d-4399-cc06-75aac7f8ec59"
      },
      "source": [
        "import json\n",
        "open('loss_list.json', 'w').write(json.dumps(loss_appearances))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "5317"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cxrsOPotEO8g"
      },
      "source": [
        "# Compare loss on multiple-token phrases and probability using ecco"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "srVMSnEvEXK0"
      },
      "source": [
        "%%capture\n",
        "! pip install ecco transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pGPho-OKEd2_"
      },
      "source": [
        "import ecco\n",
        "lm = ecco.from_pretrained('gpt2')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 162
        },
        "id": "gDI9ZOSGEpd_",
        "outputId": "83d800f3-2f87-481d-f665-d719f6756fee"
      },
      "source": [
        "output_0 = lm.generate(\"I ate an\", generate=1, do_sample=False)\n",
        "output_0.layer_predictions(position=3, layer=11)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html lang=\"en\">\n",
              "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
              "<script>\n",
              "    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
              "    // var ecco_url = 'http://localhost:8000/ecco/'\n",
              "\n",
              "    if (window.ecco === undefined) window.ecco = {}\n",
              "\n",
              "    requirejs.config({\n",
              "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
              "        nodeRequire: require,\n",
              "        paths: {\n",
              "            d3: \"https://d3js.org/d3.v5.min\",\n",
              "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
              "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
              "            // oldEcco: ecco_url + 'ecco',\n",
              "            ecco: ecco_url + 'js/0.0.4/ecco-bundle.min',\n",
              "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
              "        }\n",
              "    });\n",
              "    requirejs(['d3'],\n",
              "        function (d3) {\n",
              "            d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
              "        })\n",
              "\n",
              "    console.log('Ecco initialize.')\n",
              "</script>\n",
              "\n",
              "<head>\n",
              "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
              "</head>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<script>\n",
              "    // console.log('defining basic')\n",
              "    define('basic', ['d3'],\n",
              "        function (d3) {\n",
              "            return {\n",
              "                init: function (viz_id = null) {\n",
              "\n",
              "                    // console.log('basic init')\n",
              "                    if (viz_id == null){\n",
              "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
              "                    }\n",
              "                    const div = d3.select('#basic').attr('id', viz_id),\n",
              "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
              "\n",
              "                    // Link to CSS file\n",
              "                    d3.select(div_parent).insert('link')\n",
              "                        .attr('rel', 'stylesheet')\n",
              "                        .attr('type', 'text/css')\n",
              "                        .attr('href', ecco_url + 'html/styles.css')\n",
              "\n",
              "                    return viz_id\n",
              "                }\n",
              "            }\n",
              "        }, function (err) {console.log(err);}\n",
              "    )\n",
              "    // console.log('after defining basic')\n",
              "\n",
              "</script>\n",
              "<div id=\"basic\"></div>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "\n",
              "         requirejs( ['basic', 'ecco'], function(basic, ecco){\n",
              "            basic.init('viz_375714')\n",
              "\n",
              "            window.ecco['viz_375714'] = ecco.renderOutputSequence('viz_375714', {'tokens': [{'token': 'I', 'position': 0, 'token_id': 40, 'type': 'input'}, {'token': ' ate', 'position': 1, 'token_id': 15063, 'type': 'input'}, {'token': ' an', 'position': 2, 'token_id': 281, 'type': 'input'}]})\n",
              "         }, function (err) {\n",
              "            console.log(err);\n",
              "        })\n"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "        // We don't really need these require scripts. But this is to avert\n",
              "        //this code from running before display_input_sequence which DOES require external files\n",
              "        requirejs(['basic', 'ecco'], function(basic, ecco){\n",
              "                console.log('addToken viz_id', 'viz_375714');\n",
              "                window.ecco['viz_375714'].addToken({\"token\": \" apple\", \"token_id\": 17180, \"position\": 3, \"type\": \"output\"})\n",
              "                window.ecco['viz_375714'].redraw()\n",
              "        })\n",
              "        "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html lang=\"en\">\n",
              "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
              "<script>\n",
              "    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
              "    // var ecco_url = 'http://localhost:8000/ecco/'\n",
              "\n",
              "    if (window.ecco === undefined) window.ecco = {}\n",
              "\n",
              "    requirejs.config({\n",
              "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
              "        nodeRequire: require,\n",
              "        paths: {\n",
              "            d3: \"https://d3js.org/d3.v5.min\",\n",
              "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
              "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
              "            // oldEcco: ecco_url + 'ecco',\n",
              "            ecco: ecco_url + 'js/0.0.4/ecco-bundle.min',\n",
              "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
              "        }\n",
              "    });\n",
              "    requirejs(['d3'],\n",
              "        function (d3) {\n",
              "            d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
              "        })\n",
              "\n",
              "    console.log('Ecco initialize.')\n",
              "</script>\n",
              "\n",
              "<head>\n",
              "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
              "</head>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<script>\n",
              "    // console.log('defining basic')\n",
              "    define('basic', ['d3'],\n",
              "        function (d3) {\n",
              "            return {\n",
              "                init: function (viz_id = null) {\n",
              "\n",
              "                    // console.log('basic init')\n",
              "                    if (viz_id == null){\n",
              "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
              "                    }\n",
              "                    const div = d3.select('#basic').attr('id', viz_id),\n",
              "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
              "\n",
              "                    // Link to CSS file\n",
              "                    d3.select(div_parent).insert('link')\n",
              "                        .attr('rel', 'stylesheet')\n",
              "                        .attr('type', 'text/css')\n",
              "                        .attr('href', ecco_url + 'html/styles.css')\n",
              "\n",
              "                    return viz_id\n",
              "                }\n",
              "            }\n",
              "        }, function (err) {console.log(err);}\n",
              "    )\n",
              "    // console.log('after defining basic')\n",
              "\n",
              "</script>\n",
              "<div id=\"basic\"></div>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "         requirejs(['basic', 'ecco'], function(basic, ecco){\n",
              "            const viz_id = basic.init()\n",
              "\n",
              "\n",
              "            let pred = new ecco.LayerPredictions({\n",
              "                parentDiv: viz_id,\n",
              "                data:[[{\"token\": \" apple\", \"prob\": \"0.105445765\", \"ranking\": 1, \"layer\": 11}, {\"token\": \" entire\", \"prob\": \"0.082901835\", \"ranking\": 2, \"layer\": 11}, {\"token\": \" egg\", \"prob\": \"0.081563026\", \"ranking\": 3, \"layer\": 11}, {\"token\": \" average\", \"prob\": \"0.04114284\", \"ranking\": 4, \"layer\": 11}, {\"token\": \" awful\", \"prob\": \"0.03567305\", \"ranking\": 5, \"layer\": 11}, {\"token\": \" hour\", \"prob\": \"0.031087063\", \"ranking\": 6, \"layer\": 11}, {\"token\": \" extra\", \"prob\": \"0.026870172\", \"ranking\": 7, \"layer\": 11}, {\"token\": \" orange\", \"prob\": \"0.025671016\", \"ranking\": 8, \"layer\": 11}, {\"token\": \" enormous\", \"prob\": \"0.014124628\", \"ranking\": 9, \"layer\": 11}, {\"token\": \" avocado\", \"prob\": \"0.013964226\", \"ranking\": 10, \"layer\": 11}]]\n",
              "            })\n",
              "            pred.init()\n",
              "         }, function (err) {\n",
              "            console.log(viz_id, err);\n",
              "        })"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 162
        },
        "id": "lTSVwPthFdni",
        "outputId": "37d81eaa-4728-463d-f8c7-b0ecd8fc5d1a"
      },
      "source": [
        "output_1 = lm.generate(\"I ate an apple\", generate=1, do_sample=False)\n",
        "output_1.layer_predictions(position=4, layer=11)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html lang=\"en\">\n",
              "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
              "<script>\n",
              "    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
              "    // var ecco_url = 'http://localhost:8000/ecco/'\n",
              "\n",
              "    if (window.ecco === undefined) window.ecco = {}\n",
              "\n",
              "    requirejs.config({\n",
              "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
              "        nodeRequire: require,\n",
              "        paths: {\n",
              "            d3: \"https://d3js.org/d3.v5.min\",\n",
              "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
              "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
              "            // oldEcco: ecco_url + 'ecco',\n",
              "            ecco: ecco_url + 'js/0.0.4/ecco-bundle.min',\n",
              "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
              "        }\n",
              "    });\n",
              "    requirejs(['d3'],\n",
              "        function (d3) {\n",
              "            d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
              "        })\n",
              "\n",
              "    console.log('Ecco initialize.')\n",
              "</script>\n",
              "\n",
              "<head>\n",
              "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
              "</head>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<script>\n",
              "    // console.log('defining basic')\n",
              "    define('basic', ['d3'],\n",
              "        function (d3) {\n",
              "            return {\n",
              "                init: function (viz_id = null) {\n",
              "\n",
              "                    // console.log('basic init')\n",
              "                    if (viz_id == null){\n",
              "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
              "                    }\n",
              "                    const div = d3.select('#basic').attr('id', viz_id),\n",
              "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
              "\n",
              "                    // Link to CSS file\n",
              "                    d3.select(div_parent).insert('link')\n",
              "                        .attr('rel', 'stylesheet')\n",
              "                        .attr('type', 'text/css')\n",
              "                        .attr('href', ecco_url + 'html/styles.css')\n",
              "\n",
              "                    return viz_id\n",
              "                }\n",
              "            }\n",
              "        }, function (err) {console.log(err);}\n",
              "    )\n",
              "    // console.log('after defining basic')\n",
              "\n",
              "</script>\n",
              "<div id=\"basic\"></div>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "\n",
              "         requirejs( ['basic', 'ecco'], function(basic, ecco){\n",
              "            basic.init('viz_342788')\n",
              "\n",
              "            window.ecco['viz_342788'] = ecco.renderOutputSequence('viz_342788', {'tokens': [{'token': 'I', 'position': 0, 'token_id': 40, 'type': 'input'}, {'token': ' ate', 'position': 1, 'token_id': 15063, 'type': 'input'}, {'token': ' an', 'position': 2, 'token_id': 281, 'type': 'input'}, {'token': ' apple', 'position': 3, 'token_id': 17180, 'type': 'input'}]})\n",
              "         }, function (err) {\n",
              "            console.log(err);\n",
              "        })\n"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "        // We don't really need these require scripts. But this is to avert\n",
              "        //this code from running before display_input_sequence which DOES require external files\n",
              "        requirejs(['basic', 'ecco'], function(basic, ecco){\n",
              "                console.log('addToken viz_id', 'viz_342788');\n",
              "                window.ecco['viz_342788'].addToken({\"token\": \" and\", \"token_id\": 290, \"position\": 4, \"type\": \"output\"})\n",
              "                window.ecco['viz_342788'].redraw()\n",
              "        })\n",
              "        "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html lang=\"en\">\n",
              "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
              "<script>\n",
              "    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
              "    // var ecco_url = 'http://localhost:8000/ecco/'\n",
              "\n",
              "    if (window.ecco === undefined) window.ecco = {}\n",
              "\n",
              "    requirejs.config({\n",
              "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
              "        nodeRequire: require,\n",
              "        paths: {\n",
              "            d3: \"https://d3js.org/d3.v5.min\",\n",
              "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
              "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
              "            // oldEcco: ecco_url + 'ecco',\n",
              "            ecco: ecco_url + 'js/0.0.4/ecco-bundle.min',\n",
              "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
              "        }\n",
              "    });\n",
              "    requirejs(['d3'],\n",
              "        function (d3) {\n",
              "            d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
              "        })\n",
              "\n",
              "    console.log('Ecco initialize.')\n",
              "</script>\n",
              "\n",
              "<head>\n",
              "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
              "</head>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<script>\n",
              "    // console.log('defining basic')\n",
              "    define('basic', ['d3'],\n",
              "        function (d3) {\n",
              "            return {\n",
              "                init: function (viz_id = null) {\n",
              "\n",
              "                    // console.log('basic init')\n",
              "                    if (viz_id == null){\n",
              "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
              "                    }\n",
              "                    const div = d3.select('#basic').attr('id', viz_id),\n",
              "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
              "\n",
              "                    // Link to CSS file\n",
              "                    d3.select(div_parent).insert('link')\n",
              "                        .attr('rel', 'stylesheet')\n",
              "                        .attr('type', 'text/css')\n",
              "                        .attr('href', ecco_url + 'html/styles.css')\n",
              "\n",
              "                    return viz_id\n",
              "                }\n",
              "            }\n",
              "        }, function (err) {console.log(err);}\n",
              "    )\n",
              "    // console.log('after defining basic')\n",
              "\n",
              "</script>\n",
              "<div id=\"basic\"></div>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "         requirejs(['basic', 'ecco'], function(basic, ecco){\n",
              "            const viz_id = basic.init()\n",
              "\n",
              "\n",
              "            let pred = new ecco.LayerPredictions({\n",
              "                parentDiv: viz_id,\n",
              "                data:[[{\"token\": \" and\", \"prob\": \"0.118106425\", \"ranking\": 1, \"layer\": 11}, {\"token\": \",\", \"prob\": \"0.10378815\", \"ranking\": 2, \"layer\": 11}, {\"token\": \" pie\", \"prob\": \"0.06384794\", \"ranking\": 3, \"layer\": 11}, {\"token\": \".\", \"prob\": \"0.05152292\", \"ranking\": 4, \"layer\": 11}, {\"token\": \" a\", \"prob\": \"0.03397257\", \"ranking\": 5, \"layer\": 11}, {\"token\": \" for\", \"prob\": \"0.028119376\", \"ranking\": 6, \"layer\": 11}, {\"token\": \" with\", \"prob\": \"0.026313325\", \"ranking\": 7, \"layer\": 11}, {\"token\": \" at\", \"prob\": \"0.024098247\", \"ranking\": 8, \"layer\": 11}, {\"token\": \" in\", \"prob\": \"0.02072776\", \"ranking\": 9, \"layer\": 11}, {\"token\": \" from\", \"prob\": \"0.018730257\", \"ranking\": 10, \"layer\": 11}]]\n",
              "            })\n",
              "            pred.init()\n",
              "         }, function (err) {\n",
              "            console.log(viz_id, err);\n",
              "        })"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TRbgRV3QFt5-"
      },
      "source": [
        "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
        "model = AutoModelForCausalLM.from_pretrained('gpt2')\n",
        "tokenizer = AutoTokenizer.from_pretrained('gpt2')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VpKZT2VTF1kO"
      },
      "source": [
        "from torch import tensor\n",
        "\n",
        "def tokenized(inp):\n",
        "  inputs = tokenizer([inp])\n",
        "  inputs['input_ids'] = tensor(inputs['input_ids'])\n",
        "  inputs['attention_mask'] = tensor(inputs['attention_mask'])\n",
        "  return inputs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YD45xIwhGIoS",
        "outputId": "e601ae34-a63c-4e09-a538-7f2e6df0996e"
      },
      "source": [
        "from torch.nn import CrossEntropyLoss\n",
        "\n",
        "input1 = tokenized('I ate an apple')\n",
        "print(input1)\n",
        "output1 = model(**input1)\n",
        "output1.keys()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': tensor([[   40, 15063,   281, 17180]]), 'attention_mask': tensor([[1, 1, 1, 1]])}\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "odict_keys(['logits', 'past_key_values'])"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CbL77yd2JAvA"
      },
      "source": [
        "out_logits = output1['logits']\n",
        "out_labels = input1['input_ids']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D5oqAYRlJCdJ"
      },
      "source": [
        "focus_tokens = [[0, 0], [2, 3]]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-W-0E7D3JaUy",
        "outputId": "c34fab7e-0a13-4e39-dbec-37003686ae3f"
      },
      "source": [
        "out_logits[focus_tokens][..., :-1, :].contiguous()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-100.0585,  -98.2642, -102.5634,  ..., -104.1698, -107.1987,\n",
              "         -102.1312]], grad_fn=<SliceBackward0>)"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cYSxIK0pIpA6"
      },
      "source": [
        "shift_logits = out_logits[focus_tokens][..., :-1, :].contiguous()\n",
        "shift_labels = out_labels[focus_tokens][..., 1:].contiguous()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L5PjXK4lJSv3",
        "outputId": "39fcae25-44b2-4435-d78d-3b0b56eac2fc"
      },
      "source": [
        "loss_fct = CrossEntropyLoss(ignore_index=-1)\n",
        "loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)),\n",
        "                shift_labels.view(-1))\n",
        "loss"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(2.2496, grad_fn=<NllLossBackward0>)"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bogzjfESIOpM",
        "outputId": "2b95955a-2752-40c0-bf69-eeab1ac97a21"
      },
      "source": [
        "input2 = tokenized('I ate an apple pie')\n",
        "output2 = model(**input2)\n",
        "\n",
        "out_logits = output2['logits']\n",
        "out_labels = input2['input_ids']\n",
        "focus_tokens = [[0, 0, 0], [2, 3, 4]]\n",
        "\n",
        "shift_logits = out_logits[focus_tokens][..., :-1, :].contiguous()\n",
        "shift_labels = out_labels[focus_tokens][..., 1:].contiguous()\n",
        "\n",
        "loss_fct = CrossEntropyLoss(ignore_index=-1)\n",
        "loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)),\n",
        "                shift_labels.view(-1))\n",
        "loss"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(2.5004, grad_fn=<NllLossBackward0>)"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Am2EEToIKG9t",
        "outputId": "3af0819e-636f-4566-9efd-a54014eb4d93"
      },
      "source": [
        "input3 = tokenized('I ate an apple tree')\n",
        "output3 = model(**input3)\n",
        "\n",
        "out_logits = output3['logits']\n",
        "out_labels = input3['input_ids']\n",
        "focus_tokens = [[0, 0, 0], [2, 3, 4]]\n",
        "\n",
        "shift_logits = out_logits[focus_tokens][..., :-1, :].contiguous()\n",
        "shift_labels = out_labels[focus_tokens][..., 1:].contiguous()\n",
        "\n",
        "loss_fct = CrossEntropyLoss(ignore_index=-1)\n",
        "loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)),\n",
        "                shift_labels.view(-1))\n",
        "loss"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(4.1849, grad_fn=<NllLossBackward0>)"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HFuVoBTVONqc",
        "outputId": "2e89c768-fe35-47ac-e5d9-ecfb488b3970"
      },
      "source": [
        "input4 = tokenized('I ate an avocado')\n",
        "output4 = model(**input4)\n",
        "\n",
        "out_logits = output4['logits']\n",
        "out_labels = input4['input_ids']\n",
        "focus_tokens = [[0, 0], [2, 3]]\n",
        "\n",
        "shift_logits = out_logits[focus_tokens][..., :-1, :].contiguous()\n",
        "shift_labels = out_labels[focus_tokens][..., 1:].contiguous()\n",
        "\n",
        "loss_fct = CrossEntropyLoss(ignore_index=-1)\n",
        "loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)),\n",
        "                shift_labels.view(-1))\n",
        "loss"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(4.2713, grad_fn=<NllLossBackward0>)"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    }
  ]
}