Tokenisieren.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "93384110a33247c5af6398f4c2e2491c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2a76a86ff45b4c8e81e4b4691e4d2c3e",
              "IPY_MODEL_3f81f5615e754369b5d464dd9c073c94",
              "IPY_MODEL_0ef13b5ca6744c9891dd0b96a6fb2bce"
            ],
            "layout": "IPY_MODEL_4d4fc3a8d7614af1a5d158d73667a5aa"
          }
        },
        "2a76a86ff45b4c8e81e4b4691e4d2c3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2724865317364993985c04b64434a65a",
            "placeholder": "​",
            "style": "IPY_MODEL_141f3b4c77d54030ba014265f4ead6c7",
            "value": "config.json: 100%"
          }
        },
        "3f81f5615e754369b5d464dd9c073c94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4bed29cf9ffc48a3ae19074442d7d91a",
            "max": 665,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6040ae9e11724ea6be5417e8b042b38f",
            "value": 665
          }
        },
        "0ef13b5ca6744c9891dd0b96a6fb2bce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_306d170a9e044d9fbd8a7887dfc4fecd",
            "placeholder": "​",
            "style": "IPY_MODEL_59fcf65387974878a5a34ab8163c52c8",
            "value": " 665/665 [00:00&lt;00:00, 14.8kB/s]"
          }
        },
        "4d4fc3a8d7614af1a5d158d73667a5aa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2724865317364993985c04b64434a65a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "141f3b4c77d54030ba014265f4ead6c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4bed29cf9ffc48a3ae19074442d7d91a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6040ae9e11724ea6be5417e8b042b38f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "306d170a9e044d9fbd8a7887dfc4fecd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "59fcf65387974878a5a34ab8163c52c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6644b8a3871b45b5aa12f08d5ede5255": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e37d870b47b84c6cbbf2ebaeef20a5bf",
              "IPY_MODEL_0e922d144be64c9f9081e7d2646cdff8",
              "IPY_MODEL_2bed4f7251bc4180b898e2d6ecf846fa"
            ],
            "layout": "IPY_MODEL_bc259c120a42495985ca813802f11603"
          }
        },
        "e37d870b47b84c6cbbf2ebaeef20a5bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2524a426477e40ba9cbcf81976ea4a54",
            "placeholder": "​",
            "style": "IPY_MODEL_6118f7b7afe44cab9717fedcef5ca509",
            "value": "vocab.json: 100%"
          }
        },
        "0e922d144be64c9f9081e7d2646cdff8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3f65a21185a148069d88bcbcea3aac69",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d5a0f7b55c0b48a49b2829f20a50e423",
            "value": 1042301
          }
        },
        "2bed4f7251bc4180b898e2d6ecf846fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dc1072e0431347a1941a6ae018111da2",
            "placeholder": "​",
            "style": "IPY_MODEL_b52651854c904bd98cf9c1be9119aec7",
            "value": " 1.04M/1.04M [00:00&lt;00:00, 4.93MB/s]"
          }
        },
        "bc259c120a42495985ca813802f11603": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2524a426477e40ba9cbcf81976ea4a54": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6118f7b7afe44cab9717fedcef5ca509": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3f65a21185a148069d88bcbcea3aac69": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d5a0f7b55c0b48a49b2829f20a50e423": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "dc1072e0431347a1941a6ae018111da2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b52651854c904bd98cf9c1be9119aec7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "103210e283444f99873fb8f6ad82afb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2eefdd480e664e238c8e786ce98ba2b1",
              "IPY_MODEL_8e1f88711fb1449daf14db452cdcf9fa",
              "IPY_MODEL_2eecc76464494e5da1cd5a2709a377fd"
            ],
            "layout": "IPY_MODEL_b76ec4d730494b6f85de5cf9c933c60a"
          }
        },
        "2eefdd480e664e238c8e786ce98ba2b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_124e585744f146ddaa512d67b35b20fa",
            "placeholder": "​",
            "style": "IPY_MODEL_7a093e2e01ed4caf949f48f3fa3d4a36",
            "value": "merges.txt: 100%"
          }
        },
        "8e1f88711fb1449daf14db452cdcf9fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8dbf38330b7849e0beace333d623ca91",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_156945f6f67a4ed4974c08ab9695b3ad",
            "value": 456318
          }
        },
        "2eecc76464494e5da1cd5a2709a377fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5a94d741d6d945f087bc4cc05376734d",
            "placeholder": "​",
            "style": "IPY_MODEL_ffbc00795947413498b529a622e9c290",
            "value": " 456k/456k [00:00&lt;00:00, 6.51MB/s]"
          }
        },
        "b76ec4d730494b6f85de5cf9c933c60a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "124e585744f146ddaa512d67b35b20fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7a093e2e01ed4caf949f48f3fa3d4a36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8dbf38330b7849e0beace333d623ca91": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "156945f6f67a4ed4974c08ab9695b3ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5a94d741d6d945f087bc4cc05376734d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ffbc00795947413498b529a622e9c290": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c80030f9da34470a86df2208218d10fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_afc90d04799b49448dba0ae1a90bc83c",
              "IPY_MODEL_877d2d28bf6749d88f0fcd5f901c6806",
              "IPY_MODEL_734cb0cb07c24a899a120de8a9e9b981"
            ],
            "layout": "IPY_MODEL_554b0ce64437425797c553c8e492a0e1"
          }
        },
        "afc90d04799b49448dba0ae1a90bc83c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_962a9b1ba4b043bf813223cd9f9b8b8b",
            "placeholder": "​",
            "style": "IPY_MODEL_fe68f0b8e6c04b9793f1b6d0c529aedc",
            "value": "tokenizer.json: 100%"
          }
        },
        "877d2d28bf6749d88f0fcd5f901c6806": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_91926028ac9f431e9413570c1e03dd0d",
            "max": 1355256,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d3c272150a8245d3a5bc08aa36dd6e60",
            "value": 1355256
          }
        },
        "734cb0cb07c24a899a120de8a9e9b981": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d0452c1fd5a74729a7d295f788c5090e",
            "placeholder": "​",
            "style": "IPY_MODEL_6ecaae3e0c4b427b8655526bea356824",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 13.3MB/s]"
          }
        },
        "554b0ce64437425797c553c8e492a0e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "962a9b1ba4b043bf813223cd9f9b8b8b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fe68f0b8e6c04b9793f1b6d0c529aedc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "91926028ac9f431e9413570c1e03dd0d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d3c272150a8245d3a5bc08aa36dd6e60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d0452c1fd5a74729a7d295f788c5090e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6ecaae3e0c4b427b8655526bea356824": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jansoe/ANNet/blob/main/Tokenisieren.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9dB45UYZTUqe"
      },
      "source": [
        "# Transformers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Px2ChByN_XOI"
      },
      "source": [
        "!pip install transformers -q"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import transformers\n",
        "import torch"
      ],
      "metadata": {
        "id": "y-epkjNq3FJM"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mHY90e0aQ6ag"
      },
      "source": [
        "## Input : Tokens"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "83mPXhYNRlTY",
        "outputId": "f145db94-1e5d-440c-cae8-3e7c63d7adeb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "93384110a33247c5af6398f4c2e2491c",
            "2a76a86ff45b4c8e81e4b4691e4d2c3e",
            "3f81f5615e754369b5d464dd9c073c94",
            "0ef13b5ca6744c9891dd0b96a6fb2bce",
            "4d4fc3a8d7614af1a5d158d73667a5aa",
            "2724865317364993985c04b64434a65a",
            "141f3b4c77d54030ba014265f4ead6c7",
            "4bed29cf9ffc48a3ae19074442d7d91a",
            "6040ae9e11724ea6be5417e8b042b38f",
            "306d170a9e044d9fbd8a7887dfc4fecd",
            "59fcf65387974878a5a34ab8163c52c8",
            "6644b8a3871b45b5aa12f08d5ede5255",
            "e37d870b47b84c6cbbf2ebaeef20a5bf",
            "0e922d144be64c9f9081e7d2646cdff8",
            "2bed4f7251bc4180b898e2d6ecf846fa",
            "bc259c120a42495985ca813802f11603",
            "2524a426477e40ba9cbcf81976ea4a54",
            "6118f7b7afe44cab9717fedcef5ca509",
            "3f65a21185a148069d88bcbcea3aac69",
            "d5a0f7b55c0b48a49b2829f20a50e423",
            "dc1072e0431347a1941a6ae018111da2",
            "b52651854c904bd98cf9c1be9119aec7",
            "103210e283444f99873fb8f6ad82afb0",
            "2eefdd480e664e238c8e786ce98ba2b1",
            "8e1f88711fb1449daf14db452cdcf9fa",
            "2eecc76464494e5da1cd5a2709a377fd",
            "b76ec4d730494b6f85de5cf9c933c60a",
            "124e585744f146ddaa512d67b35b20fa",
            "7a093e2e01ed4caf949f48f3fa3d4a36",
            "8dbf38330b7849e0beace333d623ca91",
            "156945f6f67a4ed4974c08ab9695b3ad",
            "5a94d741d6d945f087bc4cc05376734d",
            "ffbc00795947413498b529a622e9c290",
            "c80030f9da34470a86df2208218d10fe",
            "afc90d04799b49448dba0ae1a90bc83c",
            "877d2d28bf6749d88f0fcd5f901c6806",
            "734cb0cb07c24a899a120de8a9e9b981",
            "554b0ce64437425797c553c8e492a0e1",
            "962a9b1ba4b043bf813223cd9f9b8b8b",
            "fe68f0b8e6c04b9793f1b6d0c529aedc",
            "91926028ac9f431e9413570c1e03dd0d",
            "d3c272150a8245d3a5bc08aa36dd6e60",
            "d0452c1fd5a74729a7d295f788c5090e",
            "6ecaae3e0c4b427b8655526bea356824"
          ]
        }
      },
      "source": [
        "#model_version = 'bert-base-uncased'\n",
        "model_version = 'gpt2'\n",
        "tokenizer = transformers.AutoTokenizer.from_pretrained(model_version)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/665 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "93384110a33247c5af6398f4c2e2491c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6644b8a3871b45b5aa12f08d5ede5255"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "103210e283444f99873fb8f6ad82afb0"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c80030f9da34470a86df2208218d10fe"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.tokenize('This is a test of the tokenizer')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OYo84qc9F4Ks",
        "outputId": "d5d15642-d5ad-4ba5-91a9-2f3cceb0b44c"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['This', 'Ġis', 'Ġa', 'Ġtest', 'Ġof', 'Ġthe', 'Ġtoken', 'izer']"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9rzPyvlEKScs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b3169680-4d2e-43d4-925f-a857cf119504"
      },
      "source": [
        "tokens = tokenizer('This is a test of the tokenizer')['input_ids']\n",
        "tokens"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1212, 318, 257, 1332, 286, 262, 11241, 7509]"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "[tokenizer.convert_ids_to_tokens(t) for t in tokens]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gg_KaTYzFvgq",
        "outputId": "f6e55be4-4a95-4c84-b8f6-28a7f7de4d74"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['This', 'Ġis', 'Ġa', 'Ġtest', 'Ġof', 'Ġthe', 'Ġtoken', 'izer']"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.decode(tokens)"
      ],
      "metadata": {
        "id": "r68IvDN5v7ck",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "09d60603-1dc3-4c2d-fa50-fc2f23ee5e15"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'This is a test of the tokenizer'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J9YhAxm_Axks",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d96049c8-8553-44bb-87ef-bb9c5fdc9780"
      },
      "source": [
        "tokenizer.tokenize('n0thing is corect')"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['n', '0', 'thing', 'Ġis', 'Ġcore', 'ct']"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.vocab"
      ],
      "metadata": {
        "id": "FBLbqQnCvVsC",
        "outputId": "5e28d580-7dc0-4349-ad1a-f70418ec1c07",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'Ġdisgruntled': 48967,\n",
              " 'Ġmorphology': 46320,\n",
              " 'Ġstatements': 6299,\n",
              " 'Kar': 37753,\n",
              " 'language': 16129,\n",
              " 'VPN': 33883,\n",
              " 'HC': 16045,\n",
              " 'IONS': 11053,\n",
              " 'ĠOther': 3819,\n",
              " 'Ġpercentile': 37894,\n",
              " 'ĠStarCraft': 38460,\n",
              " 'ĠJohn': 1757,\n",
              " 'Ġdevise': 44668,\n",
              " 'ussia': 31269,\n",
              " 'Ġsleeves': 27409,\n",
              " 'Ġreceptive': 41062,\n",
              " 'ĠRadar': 38426,\n",
              " 'angling': 27499,\n",
              " 'ĠGoose': 46317,\n",
              " 'Cole': 46509,\n",
              " 'Ġdefeating': 21320,\n",
              " 'Ġsoftened': 41534,\n",
              " 'Ġschedules': 24025,\n",
              " 'Ġhall': 6899,\n",
              " 'Ġwheat': 17135,\n",
              " 'Ġath': 5782,\n",
              " 'Ġcandid': 2960,\n",
              " 'Ġsketch': 17548,\n",
              " 'Ġswitching': 15430,\n",
              " 'Ġ(Â£': 23068,\n",
              " 'ĠLean': 45661,\n",
              " 'ĠInstant': 24470,\n",
              " 'Ġlandfill': 47712,\n",
              " 'Ġdeclined': 7392,\n",
              " 'alogue': 30326,\n",
              " 'Ġdonors': 13334,\n",
              " 'Ġrecruiting': 16517,\n",
              " 'ĠEmail': 9570,\n",
              " 'Ġdocumentaries': 43014,\n",
              " 'ueller': 16466,\n",
              " 'Ġvinegar': 26600,\n",
              " 'ĠMMR': 41362,\n",
              " 'izont': 12071,\n",
              " 'hole': 13207,\n",
              " 'Lear': 14961,\n",
              " 'except': 16341,\n",
              " 'REAM': 32235,\n",
              " '671': 46250,\n",
              " 'ĠPharmaceutical': 37175,\n",
              " 'ĠHardcore': 49691,\n",
              " 'Ġnarratives': 26274,\n",
              " 'ĠCanterbury': 43581,\n",
              " 'examination': 47779,\n",
              " 'ĠQuran': 30018,\n",
              " 'ĠEstablishment': 37132,\n",
              " 'ĠPier': 13762,\n",
              " 'ĠChair': 9369,\n",
              " 'Ġmisdemeanor': 22403,\n",
              " 'TB': 22737,\n",
              " 'tha': 12898,\n",
              " '282': 32568,\n",
              " 'Ġsecure': 5713,\n",
              " 'ĠFra': 39313,\n",
              " 'finals': 32089,\n",
              " 'Ġnationalism': 24436,\n",
              " 'ĠAuthors': 46665,\n",
              " 'ĠMOR': 35208,\n",
              " 'ynamic': 28995,\n",
              " 'ucer': 48915,\n",
              " 'ĠSeaf': 49967,\n",
              " 'Ġninth': 19646,\n",
              " 'ĠLens': 30092,\n",
              " 'Tre': 31055,\n",
              " 'Ġheroes': 10281,\n",
              " 'aff': 2001,\n",
              " 'scope': 29982,\n",
              " 'ĠAffordable': 19258,\n",
              " 'ng': 782,\n",
              " 'Ġrare': 4071,\n",
              " 'Ġproletariat': 30444,\n",
              " 'ĠHawai': 10847,\n",
              " 'Ġdeviations': 47060,\n",
              " 'ĠstrutConnector': 50009,\n",
              " 'Ġyacht': 43207,\n",
              " 'tered': 4400,\n",
              " 'ing': 278,\n",
              " 'Ġask': 1265,\n",
              " 'ĠTrad': 8397,\n",
              " 'ĠMerkel': 18263,\n",
              " 'Ġdevice': 3335,\n",
              " 'ĠSecond': 5498,\n",
              " 'Ġnightly': 37862,\n",
              " 'Ġscheduled': 7530,\n",
              " 'Ġfoundation': 8489,\n",
              " '.....': 12359,\n",
              " 'Ġaccidents': 17390,\n",
              " 'ĠIndian': 3942,\n",
              " 'Ġvalid': 4938,\n",
              " 'Ġfasting': 24845,\n",
              " 'ĠRs': 12820,\n",
              " 'alloc': 32332,\n",
              " 'Ġcommissioners': 33020,\n",
              " 'Ġdiversion': 34851,\n",
              " 'Ġdamning': 36706,\n",
              " 'ĠSatellite': 33530,\n",
              " 'ĠHarold': 23987,\n",
              " 'ĠNaj': 37135,\n",
              " 'ĠIllusion': 37697,\n",
              " 'Ġworship': 11892,\n",
              " 'kHz': 44191,\n",
              " ']+': 48688,\n",
              " 'ĠCandle': 44973,\n",
              " 'Ġprecon': 49394,\n",
              " 'Paris': 40313,\n",
              " 'Ġprotection': 4800,\n",
              " 'resses': 16746,\n",
              " 'Ġseed': 9403,\n",
              " 'sofar': 38649,\n",
              " 'ĠEfficiency': 45728,\n",
              " 'ĠHughes': 19712,\n",
              " 'Ġwallet': 13008,\n",
              " 'ĠSwed': 7289,\n",
              " 'Ġfoc': 2133,\n",
              " 'Ġmedium': 7090,\n",
              " '556': 37864,\n",
              " 'wark': 48542,\n",
              " 'empt': 1791,\n",
              " 'Ġcirculating': 24495,\n",
              " 'ĠTBD': 34343,\n",
              " 'days': 12545,\n",
              " 'anton': 23026,\n",
              " 'Attempt': 37177,\n",
              " '035': 44215,\n",
              " 'Ġboulder': 46483,\n",
              " 'Ġleaked': 14109,\n",
              " 'Ġcleansing': 32784,\n",
              " 'Rest': 19452,\n",
              " 'Ġchanting': 31138,\n",
              " 'ĠOaks': 44781,\n",
              " 'hist': 10034,\n",
              " 'Ġhelpers': 49385,\n",
              " 'Ġskysc': 37885,\n",
              " 'ĠSam': 3409,\n",
              " 'Ġfem': 2796,\n",
              " 'mite': 32937,\n",
              " 'ause': 682,\n",
              " 'ĠCavs': 38943,\n",
              " 'Ġbumps': 34029,\n",
              " 'ĠResolution': 22406,\n",
              " 'ĠAFC': 27987,\n",
              " 'inventory': 24807,\n",
              " 'ĠBrooks': 17704,\n",
              " 'ĠHistoric': 30426,\n",
              " 'ĠLith': 19730,\n",
              " 'ĠConquer': 40963,\n",
              " 'REP': 35316,\n",
              " 'Builder': 32875,\n",
              " 'Ġfavor': 2661,\n",
              " 'Ġallegiance': 25696,\n",
              " 'ĠCoc': 18490,\n",
              " 'Ġhardened': 33034,\n",
              " 'ĠMist': 15078,\n",
              " 'Ġnegro': 47255,\n",
              " 'ĠChilean': 48014,\n",
              " 'ĠMeet': 21167,\n",
              " '42': 3682,\n",
              " 'Ess': 29508,\n",
              " 'ĠMot': 6543,\n",
              " 'Ġrecognizable': 30264,\n",
              " 'Ġruns': 4539,\n",
              " 'Ġrockets': 20293,\n",
              " 'Ġgreat': 1049,\n",
              " 'Ġfaculty': 12829,\n",
              " 'Ġvulgar': 31016,\n",
              " 'ĠElite': 16158,\n",
              " 'itect': 5712,\n",
              " 'Ġweakened': 24135,\n",
              " 'Ġ324': 38595,\n",
              " 'Ġsuffice': 34302,\n",
              " 'ĠTransformers': 39185,\n",
              " 'felt': 31985,\n",
              " 'Pic': 39507,\n",
              " 'exempt': 42679,\n",
              " 'ĠMcH': 49634,\n",
              " 'ĠLafayette': 41388,\n",
              " 'âĢ¦': 1399,\n",
              " 'Ġyelling': 22187,\n",
              " 'ield': 1164,\n",
              " 'ĠMarine': 11000,\n",
              " 'ĠGall': 7096,\n",
              " 'Ġtrips': 13229,\n",
              " 'Block': 12235,\n",
              " 'Again': 15316,\n",
              " 'Ġdisg': 13757,\n",
              " 'Available': 10493,\n",
              " 'ipes': 18636,\n",
              " 'opers': 20618,\n",
              " 'Prop': 24331,\n",
              " 'Ġtransforming': 25449,\n",
              " 'Ġendanger': 35276,\n",
              " 'Ġdece': 12738,\n",
              " 'Ġprinces': 42676,\n",
              " 'haus': 30404,\n",
              " 'ĠCountry': 12946,\n",
              " 'ĠEnemy': 21785,\n",
              " 'Ġslic': 14369,\n",
              " 'benefit': 48649,\n",
              " 'ĠVigil': 39840,\n",
              " 'inst': 8625,\n",
              " 'Ġclan': 19065,\n",
              " 'ĠMU': 25108,\n",
              " 'Ġaffecting': 13891,\n",
              " 'Ġgeographic': 22987,\n",
              " 'ĠKeys': 26363,\n",
              " 'Ġmarkers': 19736,\n",
              " 'Ġ).': 6739,\n",
              " 'let': 1616,\n",
              " 'ĠBol': 10797,\n",
              " 'Ġdrug': 2563,\n",
              " 'burg': 7423,\n",
              " 'tein': 22006,\n",
              " 'Ġrebuke': 45830,\n",
              " 'EF': 25425,\n",
              " '23': 1954,\n",
              " 'Ġfrogs': 37475,\n",
              " 'Ġearnest': 23176,\n",
              " 'Ġstaying': 10589,\n",
              " 'Ġshorthand': 45883,\n",
              " 'ĠFreddy': 45437,\n",
              " 'Ġlooms': 46234,\n",
              " 'ĠTreat': 11217,\n",
              " 'ĠRD': 31475,\n",
              " 'ĠGriffith': 30469,\n",
              " 'Ġhump': 49779,\n",
              " 'ĠHig': 11951,\n",
              " 'testers': 27205,\n",
              " 'FIN': 20032,\n",
              " 'ĠBundes': 33816,\n",
              " 'Ġsprinkle': 45799,\n",
              " 'Ġab': 450,\n",
              " 'Ġbinaries': 38640,\n",
              " 'Ġgoodies': 39863,\n",
              " 'Ġservicing': 43346,\n",
              " 'Sold': 33873,\n",
              " 'ism': 1042,\n",
              " 'Romney': 42184,\n",
              " 'Ġagility': 33546,\n",
              " 'Ġdemonstrations': 18721,\n",
              " 'ĠPep': 24226,\n",
              " 'Ġskeptical': 17988,\n",
              " 'ĠOPS': 40490,\n",
              " 'Ġcrowded': 18012,\n",
              " 'Ġnorth': 5093,\n",
              " 'ĠDirt': 49124,\n",
              " 'ĠSocrates': 41242,\n",
              " 'Ġneighborhood': 6232,\n",
              " 'ĠMEN': 41597,\n",
              " 'ĠMuscle': 43683,\n",
              " 'Ġbrainstorm': 48733,\n",
              " 'Sorry': 14385,\n",
              " 'ATING': 33881,\n",
              " 'Ġcritics': 9188,\n",
              " 'Ġflashlight': 38371,\n",
              " 'ĠPatients': 28021,\n",
              " 'ĠChu': 25459,\n",
              " 'Ġpi': 31028,\n",
              " 'ĠDip': 43945,\n",
              " 'Ġremarked': 24998,\n",
              " 'Ġbeing': 852,\n",
              " 'Ġglossy': 40551,\n",
              " 'bc': 15630,\n",
              " 'ãĥĥãĥĪ': 35799,\n",
              " '/.': 11757,\n",
              " 'Safe': 31511,\n",
              " 'ĠÏ': 18074,\n",
              " 'replace': 33491,\n",
              " 'ovember': 3239,\n",
              " 'ĠEAR': 31834,\n",
              " 'Ġdaemon': 33386,\n",
              " 'Journal': 25296,\n",
              " 'ĠLu': 6026,\n",
              " 'Ġcrotch': 45885,\n",
              " 'ifled': 47157,\n",
              " 'akening': 18800,\n",
              " 'ĠObjects': 35832,\n",
              " 'were': 22474,\n",
              " 'Ġsew': 34249,\n",
              " 'volt': 37764,\n",
              " 'Ġceiling': 13387,\n",
              " 'Ġcases': 2663,\n",
              " 'Ġinterns': 47266,\n",
              " 'Ġirregularities': 48302,\n",
              " 'Ġmel': 7758,\n",
              " 'serving': 31293,\n",
              " 'Ġpaperback': 46771,\n",
              " 'hero': 11718,\n",
              " 'inqu': 18934,\n",
              " 'nant': 22057,\n",
              " 'ĠXML': 23735,\n",
              " 'ĠPercy': 38506,\n",
              " 'until': 28446,\n",
              " 'Ġnotor': 23305,\n",
              " 'Nin': 36091,\n",
              " 'Ġapple': 17180,\n",
              " 'fitted': 38631,\n",
              " 'Ġidle': 21696,\n",
              " 'enes': 18719,\n",
              " 'hound': 39047,\n",
              " 'Ġhonored': 21014,\n",
              " 'Ġorganizational': 26564,\n",
              " 'ĠComposite': 49355,\n",
              " 'Ġestim': 3959,\n",
              " 'Phys': 43215,\n",
              " 'itative': 12464,\n",
              " 'Ġsubsid': 7905,\n",
              " 'Ġcave': 11527,\n",
              " 'ĠProtective': 45004,\n",
              " 'Ġallegations': 7636,\n",
              " 'Ġkidney': 21919,\n",
              " 'ĠINFO': 24890,\n",
              " 'Ġnonexistent': 42601,\n",
              " 'Ġsizeable': 47225,\n",
              " 'Ġdecor': 11705,\n",
              " 'itations': 20597,\n",
              " 'ĠAppro': 20010,\n",
              " 'asp': 5126,\n",
              " 'lasses': 28958,\n",
              " 'lement': 1732,\n",
              " 'ĠChak': 46620,\n",
              " 'Ġpedd': 36145,\n",
              " 'ď': 203,\n",
              " 'ports': 3742,\n",
              " 'Ġrunning': 2491,\n",
              " 'ĠTrip': 18383,\n",
              " 'Ġenlarge': 27475,\n",
              " 'water': 7050,\n",
              " 'ĠSomali': 30886,\n",
              " 'Ġaspir': 18808,\n",
              " 'Soc': 37949,\n",
              " 'ak': 461,\n",
              " 'Ġtainted': 36912,\n",
              " 'ĠLeaving': 38068,\n",
              " 'ĠPledge': 45737,\n",
              " 'ĠTemp': 24189,\n",
              " 'bool': 30388,\n",
              " 'assuming': 32935,\n",
              " 'Ġwip': 14763,\n",
              " 'Ver': 13414,\n",
              " 'irmation': 36241,\n",
              " 'ĠUri': 46688,\n",
              " 'Ġvetoed': 49581,\n",
              " 'ĠSik': 32615,\n",
              " 'ĠHillary': 5073,\n",
              " 'Ġfont': 10369,\n",
              " 'ĠAttorney': 8123,\n",
              " 'Ap': 25189,\n",
              " 'Ġunilaterally': 44486,\n",
              " 'Ġsmaller': 4833,\n",
              " 'ĠDig': 7367,\n",
              " 'ĠGarcia': 18555,\n",
              " 'Ġsurreal': 28201,\n",
              " 'ĠFlan': 39016,\n",
              " 'ø': 180,\n",
              " 'ċ': 199,\n",
              " 'Ġpercentages': 28071,\n",
              " 'Ġdrastically': 22188,\n",
              " 'oubted': 15973,\n",
              " '115': 15363,\n",
              " 'kish': 31501,\n",
              " 'Ġinvent': 8067,\n",
              " 'Ġlistened': 16399,\n",
              " 'Ġentails': 35445,\n",
              " 'Ġreservoir': 22115,\n",
              " 'gap': 43554,\n",
              " 'lex': 2588,\n",
              " 'Chinese': 23604,\n",
              " '))': 4008,\n",
              " 'ĠNAT': 10149,\n",
              " 'Ġtracts': 42385,\n",
              " 'ĠURI': 43975,\n",
              " 'Ġattorney': 6136,\n",
              " 'prefix': 40290,\n",
              " 'Fuel': 42663,\n",
              " 'ĠElon': 32451,\n",
              " 'Ġblacks': 15102,\n",
              " 'ĠWRITE': 44423,\n",
              " 'Cele': 42741,\n",
              " 'Ġcompress': 27413,\n",
              " 'Ġunfolding': 34502,\n",
              " 'Ġbinds': 37354,\n",
              " 'Ġconduit': 44457,\n",
              " 'ĠPerry': 14105,\n",
              " 'Ġstorefront': 45527,\n",
              " 'jon': 46286,\n",
              " 'Ġliving': 2877,\n",
              " '613': 47512,\n",
              " 'Ġpermission': 7170,\n",
              " 'Ġshoots': 20611,\n",
              " 'ĠIncome': 19003,\n",
              " 'Ġissuing': 19089,\n",
              " 'Ġpicture': 4286,\n",
              " 'IER': 38311,\n",
              " 'Ġqualifier': 39265,\n",
              " 'ĠDion': 29628,\n",
              " 'Ġbicycl': 48094,\n",
              " 'Ġfulfill': 14658,\n",
              " 'Ġarchaeological': 30971,\n",
              " 'ĠMelody': 38906,\n",
              " 'ĠDrama': 30647,\n",
              " 'Ġviewers': 10209,\n",
              " 'ĠSaving': 34689,\n",
              " 'Ohio': 31274,\n",
              " 'FUN': 42296,\n",
              " 'Ġworn': 12666,\n",
              " 'ĠAr': 943,\n",
              " 'iration': 15297,\n",
              " 'Ġwarrior': 16491,\n",
              " 'Ġimplicit': 16992,\n",
              " 'ĠProtest': 17801,\n",
              " 'Ġvolunteer': 13904,\n",
              " 'Ġblows': 20385,\n",
              " 'ĠThing': 21561,\n",
              " 'exper': 23100,\n",
              " 'ĠBRE': 29377,\n",
              " 'ĠRomance': 36555,\n",
              " 'CW': 43538,\n",
              " 'ĠGPA': 45283,\n",
              " '------------------------------------------------': 47232,\n",
              " 'painted': 47351,\n",
              " 'Ġviewed': 9569,\n",
              " 'BILL': 39888,\n",
              " 'Ġinnate': 28690,\n",
              " 'ĠIT': 7283,\n",
              " '458': 29334,\n",
              " 'Ġone': 530,\n",
              " 'Ġes': 1658,\n",
              " 'Ġalienated': 41258,\n",
              " 'ĠPopulation': 20133,\n",
              " 'detail': 49170,\n",
              " 'Ġfraught': 39730,\n",
              " 'STAT': 35744,\n",
              " 'Ġentering': 8218,\n",
              " 'Ġcircled': 48149,\n",
              " 'Ġskulls': 38556,\n",
              " 'Ġcoordinating': 35449,\n",
              " '1968': 42246,\n",
              " 'mington': 39773,\n",
              " '240': 16102,\n",
              " 'sing': 12215,\n",
              " '730': 43916,\n",
              " 'ĠRecomm': 19237,\n",
              " 'Ġgreen': 4077,\n",
              " 'ulating': 8306,\n",
              " 'ĠPract': 13672,\n",
              " 'ĠWeak': 28788,\n",
              " 'Ġcontain': 3994,\n",
              " 'ky': 2584,\n",
              " 'econom': 13926,\n",
              " 'hill': 12639,\n",
              " 'ĠAnniversary': 21967,\n",
              " 'ĠStr': 4285,\n",
              " 'ourney': 5604,\n",
              " 'ĠRose': 8049,\n",
              " 'Ġdigest': 16274,\n",
              " 'Ġaccessing': 22534,\n",
              " 'taking': 26103,\n",
              " 'Ġfloated': 29742,\n",
              " 'Ġproducing': 9194,\n",
              " 'ãĤĮ': 39258,\n",
              " 'UCK': 16696,\n",
              " 'Ġ1984': 12844,\n",
              " 'bda': 43444,\n",
              " 'ĠBAT': 37421,\n",
              " 'ĠWhat': 1867,\n",
              " 'Ġarm': 3211,\n",
              " 'app': 1324,\n",
              " 'ĠStarter': 37460,\n",
              " 'ĠLena': 44131,\n",
              " 'Ġfateful': 46873,\n",
              " 'Champ': 48507,\n",
              " 'Ġfootball': 4346,\n",
              " 'ĠGuatemala': 32183,\n",
              " 'emphasis': 36663,\n",
              " 'Center': 23656,\n",
              " 'debug': 24442,\n",
              " 'ĠPope': 13258,\n",
              " 'Ġreferences': 10288,\n",
              " 'Ġpursuing': 15461,\n",
              " 'ears': 4127,\n",
              " 'Ġdive': 15647,\n",
              " 'ĠPant': 11427,\n",
              " 'ĠREUTERS': 15862,\n",
              " '¿': 123,\n",
              " 'iker': 18320,\n",
              " 'Ġworm': 23849,\n",
              " 'Ġmedals': 28057,\n",
              " 'ĠInstr': 27901,\n",
              " 'Ġbreed': 15939,\n",
              " 'Ġmind': 2000,\n",
              " 'mber': 1916,\n",
              " 'Ġcomparisons': 17909,\n",
              " 'Ġessentially': 6986,\n",
              " 'esar': 18964,\n",
              " 'function': 8818,\n",
              " 'Inv': 19904,\n",
              " 'Ġswear': 21192,\n",
              " 'meter': 27231,\n",
              " 'ĠHomer': 28440,\n",
              " 'Ġcapac': 18457,\n",
              " 'rosso': 21074,\n",
              " 'uctor': 33029,\n",
              " 'Ġchildren': 1751,\n",
              " 'Wow': 22017,\n",
              " 'ĠRunning': 18162,\n",
              " 'ĠBed': 15585,\n",
              " 'Y': 56,\n",
              " 'Ġaccumulated': 22425,\n",
              " 'Ġvou': 25211,\n",
              " 'Ġd': 288,\n",
              " 'Ġsubmit': 9199,\n",
              " 'Ġstalking': 34683,\n",
              " 'Ġcharismatic': 34996,\n",
              " 'Ġsmear': 35180,\n",
              " 'Ġfro': 8400,\n",
              " 'Äĩ': 38325,\n",
              " 'ĠHemisphere': 42811,\n",
              " 'Ġfulfillment': 32402,\n",
              " 'ĠMen': 6065,\n",
              " 'Ġtextbooks': 31814,\n",
              " 'Advertisement': 4723,\n",
              " 'ĠCO': 7375,\n",
              " 'ĠKab': 28793,\n",
              " 'Ġslut': 33526,\n",
              " 'ĠChanging': 33680,\n",
              " 'Call': 14134,\n",
              " 'Ġabolished': 32424,\n",
              " 'Ġexpended': 37328,\n",
              " 'plugins': 37390,\n",
              " 'hub': 40140,\n",
              " 'vill': 41082,\n",
              " 'ĠNaw': 41179,\n",
              " 'cake': 30560,\n",
              " 'Folder': 41092,\n",
              " 'ensions': 5736,\n",
              " 'Ġcollaborating': 38152,\n",
              " 'Ġperiodically': 26034,\n",
              " 'Ġunsigned': 22165,\n",
              " 'Ġmockery': 42935,\n",
              " 'ĠSab': 9910,\n",
              " 'XM': 37643,\n",
              " 'ĠLionel': 44286,\n",
              " 'Ġcardio': 47926,\n",
              " 'Ġshenanigans': 48949,\n",
              " 'raised': 49309,\n",
              " 'ĠInventory': 35772,\n",
              " 'Ġ325': 29524,\n",
              " 'Ġmorphine': 43948,\n",
              " 'ĠLair': 15067,\n",
              " 'ĠEither': 15467,\n",
              " 'ĠWhole': 23431,\n",
              " 'ĠTill': 17888,\n",
              " 'ĠISI': 22048,\n",
              " '644': 29173,\n",
              " 'lene': 29466,\n",
              " 'Ġatten': 31919,\n",
              " 'Ġ04': 8702,\n",
              " 'ĠSixth': 28980,\n",
              " 'thritis': 34043,\n",
              " 'ĠMorrow': 34693,\n",
              " 'Ġbranding': 24150,\n",
              " 'Sah': 32194,\n",
              " 'Ġsmashed': 23354,\n",
              " 'ĠReserved': 33876,\n",
              " 'ĠLambert': 36978,\n",
              " 'ĠUk': 5065,\n",
              " 'ĠSales': 17329,\n",
              " 'Ġpatri': 12650,\n",
              " 'different': 39799,\n",
              " 'QB': 40291,\n",
              " 'Kal': 41428,\n",
              " 'ĠBurning': 21460,\n",
              " 'facing': 29532,\n",
              " '-+': 19529,\n",
              " 'ĠSamp': 42168,\n",
              " 'ĠForest': 9115,\n",
              " 'ĠARE': 15986,\n",
              " 'ĠNone': 6045,\n",
              " 'Basic': 26416,\n",
              " 'ĠYe': 11609,\n",
              " 'rend': 10920,\n",
              " 'z': 89,\n",
              " 'Ġinvaded': 26466,\n",
              " 'Ġexhaustive': 36049,\n",
              " 'ĠWebb': 25130,\n",
              " 'Ġgenius': 15632,\n",
              " 'ĠOst': 38919,\n",
              " 'ĠWah': 35893,\n",
              " 'Ġ414': 45900,\n",
              " 'ley': 1636,\n",
              " 'Additional': 17699,\n",
              " 'ĠWorm': 28505,\n",
              " 'ĠDG': 46133,\n",
              " 'ĠAless': 47319,\n",
              " 'NP': 22182,\n",
              " 'Ġpony': 26902,\n",
              " 'Ġguardians': 32769,\n",
              " 'Ġoutlet': 16615,\n",
              " 'olly': 5098,\n",
              " 'Ġconsumes': 37225,\n",
              " 'Ġ(+': 11502,\n",
              " 'Ġaliens': 16269,\n",
              " 'Ġacute': 14352,\n",
              " 'Farm': 48412,\n",
              " 'Ġstadiums': 37924,\n",
              " 'ĠCrossing': 31910,\n",
              " 'iece': 8535,\n",
              " '027': 44698,\n",
              " 'Ġseafood': 31473,\n",
              " 'Ġsucks': 22523,\n",
              " 'Ġlegislature': 15928,\n",
              " 'eco': 47704,\n",
              " 'Ġshifts': 15381,\n",
              " 'Ġprohibiting': 30363,\n",
              " 'Ġflank': 30172,\n",
              " 'ĠSpan': 49101,\n",
              " 'Ġ2021': 33448,\n",
              " 'ampa': 13299,\n",
              " 'Ġconstituencies': 43742,\n",
              " 'Ġindictment': 19975,\n",
              " 'Ġexplanatory': 44742,\n",
              " 'Ġbundle': 18537,\n",
              " 'ĠGast': 31486,\n",
              " 'ĠConcepts': 50053,\n",
              " 'rier': 5277,\n",
              " 'Ġunmatched': 48621,\n",
              " 'ĠData': 6060,\n",
              " '348': 28978,\n",
              " 'very': 548,\n",
              " 'Ġposit': 46436,\n",
              " 'ĠUSL': 44072,\n",
              " 'Ġorigins': 15587,\n",
              " 'IST': 8808,\n",
              " 'Ġyears': 812,\n",
              " 'ĠScroll': 17428,\n",
              " 'get': 1136,\n",
              " 'Ġpenetrate': 28302,\n",
              " 'Ġappe': 35787,\n",
              " 'dropping': 37554,\n",
              " 'secondary': 38238,\n",
              " 'Ġbil': 47027,\n",
              " 'ĠTick': 46093,\n",
              " 'sov': 47272,\n",
              " 'ĠCox': 18014,\n",
              " 'Ġprincipled': 48070,\n",
              " 'Ġmaster': 4958,\n",
              " 'His': 6653,\n",
              " 'Ġnick': 14428,\n",
              " 'country': 19315,\n",
              " 'eros': 27498,\n",
              " 'ĠAchilles': 39561,\n",
              " 'Ġcohorts': 40182,\n",
              " 'eller': 12368,\n",
              " 'Ġpredictably': 45741,\n",
              " 'Ġprincip': 26303,\n",
              " 'ĠRoyale': 49982,\n",
              " 'Ġadren': 26999,\n",
              " 'Ġinternal': 5387,\n",
              " 'Ġstirred': 33091,\n",
              " 'oler': 13625,\n",
              " 'ĠFollowing': 14207,\n",
              " 'Ġribbon': 29092,\n",
              " 'ĠStartup': 40472,\n",
              " 'ĠAlas': 45315,\n",
              " 'arr': 3258,\n",
              " 'liness': 26061,\n",
              " 'ccording': 2941,\n",
              " 'Ġinflammatory': 23760,\n",
              " 'OLOG': 33462,\n",
              " 'Ġdispose': 34291,\n",
              " 'light': 2971,\n",
              " 'Ġassorted': 46603,\n",
              " 'ĠWaiting': 39669,\n",
              " 'Ġperpendicular': 47190,\n",
              " '=]': 48874,\n",
              " 'Ġguid': 10103,\n",
              " 'Ġrip': 18344,\n",
              " 'Ġannexation': 41329,\n",
              " 'Ġinterviewer': 39877,\n",
              " 'Ġprinciples': 7811,\n",
              " 'ĠDemon': 7814,\n",
              " 'Ġ115': 12279,\n",
              " 'integer': 41433,\n",
              " 'Ġbrilliant': 10457,\n",
              " 'Ġeth': 4555,\n",
              " 'ĠFrance': 4881,\n",
              " 'Address': 20231,\n",
              " 'ĠMonaco': 31630,\n",
              " 'Ġrarity': 32823,\n",
              " 'Ġaiding': 37221,\n",
              " 'Ġviolet': 45725,\n",
              " 'anks': 2283,\n",
              " 'ishing': 3929,\n",
              " 'ĠCT': 16356,\n",
              " 'ĠWhenever': 21326,\n",
              " 'Calling': 48593,\n",
              " 'Ġsince': 1201,\n",
              " 'tyard': 30308,\n",
              " 'Topic': 33221,\n",
              " 'ĠStellar': 39336,\n",
              " 'Advertisements': 14592,\n",
              " 'Ġregretted': 44652,\n",
              " 'encers': 42288,\n",
              " 'empl': 18856,\n",
              " 'Moon': 31640,\n",
              " 'Ġcomic': 9048,\n",
              " 'ĠMermaid': 45523,\n",
              " 'ĠRights': 6923,\n",
              " 'az': 1031,\n",
              " 'episode': 38668,\n",
              " 'ĠRoy': 9817,\n",
              " 'Ġbc': 47125,\n",
              " 'ENTS': 15365,\n",
              " 'ĠFlash': 9973,\n",
              " 'nih': 37373,\n",
              " 'Ġdrawing': 8263,\n",
              " 'ĠCosts': 36845,\n",
              " 'ĠRape': 35286,\n",
              " 'ly': 306,\n",
              " 'OVER': 41983,\n",
              " 'ĠALS': 27249,\n",
              " 'Ã¡s': 40138,\n",
              " 'Ġrefreshed': 47193,\n",
              " 'Ġspirituality': 39409,\n",
              " 'iculture': 47428,\n",
              " 'riger': 18096,\n",
              " 'Ret': 9781,\n",
              " 'Welcome': 14618,\n",
              " '603': 35642,\n",
              " 'Ġcrisp': 23453,\n",
              " 'ĠHarm': 17925,\n",
              " 'Discover': 44596,\n",
              " 'Ġinsider': 25851,\n",
              " 'Ġshun': 32814,\n",
              " 'Ġpathways': 22963,\n",
              " 'Ġwage': 7699,\n",
              " 'ĠGord': 39592,\n",
              " 'Know': 23812,\n",
              " 'Ġconfused': 10416,\n",
              " 'ĠHampton': 36852,\n",
              " 'Ñı': 40623,\n",
              " 'Ġdoubtful': 31608,\n",
              " 'Ġinterruption': 41728,\n",
              " 'Ġreconcil': 40534,\n",
              " 'Ġangrily': 34425,\n",
              " 'Ġcreating': 4441,\n",
              " 'ĠSE': 7946,\n",
              " 'Ġdefendant': 11304,\n",
              " 'Ġstorytelling': 23689,\n",
              " 'Ġdeficient': 40043,\n",
              " 'ĠReplace': 40177,\n",
              " 'Ġelectron': 11538,\n",
              " 'ĠNazis': 19147,\n",
              " 'Ġdemocracies': 40416,\n",
              " 'meric': 946,\n",
              " 'ĠEduc': 7005,\n",
              " 'Ther': 35048,\n",
              " 'ĠUDP': 36428,\n",
              " 'Ġeducated': 15657,\n",
              " 'Ġvacated': 46346,\n",
              " 'Ġtheorem': 44728,\n",
              " 'Ġinterchange': 26478,\n",
              " 'ĠXV': 29922,\n",
              " 'Ġfolded': 24650,\n",
              " 'Ġfer': 11354,\n",
              " 'verning': 13974,\n",
              " 'ĠTalent': 29505,\n",
              " 'ĠEcc': 38308,\n",
              " 'irection': 4154,\n",
              " 'Mac': 14155,\n",
              " 'packing': 41291,\n",
              " 'afe': 8635,\n",
              " 'Mc': 9742,\n",
              " 'Ġbar': 2318,\n",
              " 'aries': 3166,\n",
              " 'Ġstagnant': 41391,\n",
              " 'Ns': 47503,\n",
              " 'ĠPaper': 14962,\n",
              " 'ĠSummit': 20014,\n",
              " 'Ġunpredict': 20378,\n",
              " 'osaurs': 22344,\n",
              " 'Ġcurrencies': 19247,\n",
              " 'vp': 36133,\n",
              " 'andering': 42454,\n",
              " 'Ġreactions': 12737,\n",
              " 'ĠAustria': 17322,\n",
              " 'Ġmisuse': 29169,\n",
              " 'Today': 8888,\n",
              " 'Davis': 36462,\n",
              " 'Tra': 15721,\n",
              " 'ĠAreas': 38662,\n",
              " 'angan': 37089,\n",
              " 'Ġpoliceman': 32166,\n",
              " 'Http': 43481,\n",
              " 'public': 11377,\n",
              " 'SO': 15821,\n",
              " '714': 45722,\n",
              " 'ĠRET': 30826,\n",
              " 'Ġvict': 2210,\n",
              " 'Ġgraz': 32703,\n",
              " 'Yeah': 10995,\n",
              " 'des': 8906,\n",
              " 'Ġencouraged': 10085,\n",
              " 'ummer': 31647,\n",
              " 'adem': 36920,\n",
              " 'Ġvisit': 3187,\n",
              " 'Edge': 37021,\n",
              " 'Ġaest': 14606,\n",
              " 'Thom': 37582,\n",
              " 'ĠCasting': 39285,\n",
              " 'ĠMarlins': 38134,\n",
              " 'Directory': 43055,\n",
              " 'CHO': 44899,\n",
              " 'Ġstip': 22111,\n",
              " 'Ġlun': 14678,\n",
              " 'å': 161,\n",
              " 'arsh': 5406,\n",
              " 'egu': 15703,\n",
              " '113': 16616,\n",
              " 'Ġaspire': 42926,\n",
              " 'ĠâĹı': 48049,\n",
              " 'ognitive': 46610,\n",
              " 'ĠColon': 14049,\n",
              " 'should': 21754,\n",
              " 'Years': 40630,\n",
              " '(&': 39434,\n",
              " 'UM': 5883,\n",
              " '213': 26427,\n",
              " 'Â©': 16224,\n",
              " 'pos': 1930,\n",
              " 'oked': 6545,\n",
              " 'Ġanimated': 15108,\n",
              " 'Ġspores': 47306,\n",
              " 'ĠMight': 24213,\n",
              " 'Ġarch': 3934,\n",
              " 'ussian': 31562,\n",
              " 'ĠClosing': 47055,\n",
              " 'elvet': 32667,\n",
              " 'Ġresearcher': 13453,\n",
              " 'VEN': 28290,\n",
              " 'ĠLah': 30693,\n",
              " 'ĠâĢĶ': 851,\n",
              " 'ĠGors': 34926,\n",
              " 'trump': 40954,\n",
              " 'ĠEuph': 47441,\n",
              " 'Ġuber': 48110,\n",
              " 'ĠGD': 27044,\n",
              " 'Stud': 13007,\n",
              " 'ĠLamp': 28607,\n",
              " 'Ġhood': 14263,\n",
              " 'ĠHanson': 35692,\n",
              " 'Ġawful': 12659,\n",
              " 'Ġfile': 2393,\n",
              " 'Ġtallest': 38760,\n",
              " 'Children': 26829,\n",
              " 'Ġcanonical': 40091,\n",
              " 'Ġgraphical': 27831,\n",
              " 'Ġ415': 40643,\n",
              " 'Otherwise': 48059,\n",
              " 'ĠGenocide': 48721,\n",
              " 'ĠEffect': 7896,\n",
              " 'Cast': 19248,\n",
              " 'Ġmartyr': 28009,\n",
              " 'ĠNort': 15209,\n",
              " 'Ġtherapists': 39395,\n",
              " '!).': 19588,\n",
              " 'ĠLone': 23405,\n",
              " 'Ġprominently': 31015,\n",
              " 'Gordon': 47073,\n",
              " 'Ġmeticulous': 47628,\n",
              " 'Ġspecifics': 23514,\n",
              " 'ĠAware': 32828,\n",
              " 'Alt': 29161,\n",
              " 'Ġmos': 10867,\n",
              " 'Fortunately': 31276,\n",
              " 'Ġstewards': 38030,\n",
              " 'ĠYards': 42673,\n",
              " 'CON': 10943,\n",
              " 'TE': 9328,\n",
              " 'HAHAHAHA': 39021,\n",
              " 'Woman': 48081,\n",
              " 'lord': 10572,\n",
              " 'boat': 24482,\n",
              " 'Ba': 34458,\n",
              " 'Ġairs': 37112,\n",
              " 'Ġquer': 42517,\n",
              " 'sufficient': 46790,\n",
              " 'Ġunderstandably': 36250,\n",
              " 'Ġmascara': 48430,\n",
              " 'Ġcortisol': 41547,\n",
              " '046': 45438,\n",
              " 'Ġbestselling': 47791,\n",
              " 'entially': 3746,\n",
              " 'Ġmillenn': 15755,\n",
              " 'ĠThey': 1119,\n",
              " 'Brazil': 39190,\n",
              " 'ĠPolicy': 7820,\n",
              " '747': 48882,\n",
              " 'Ġpraised': 15342,\n",
              " '1500': 33698,\n",
              " 'ĠHeader': 48900,\n",
              " 'ertodd': 36481,\n",
              " 'Ġabsorb': 17565,\n",
              " 'Ġhoused': 23707,\n",
              " 'ĠBrun': 15700,\n",
              " 'ĠRP': 25812,\n",
              " 'Ġshortstop': 40388,\n",
              " 'Ġ[*': 30138,\n",
              " 'ĠShy': 36838,\n",
              " 'Ġdeployments': 40854,\n",
              " 'IRO': 43708,\n",
              " 'Ġsem': 5026,\n",
              " 'Ġpaths': 13532,\n",
              " 'Ġpoll': 3278,\n",
              " 'pract': 29152,\n",
              " 'Palestinian': 35969,\n",
              " 'ĠBal': 8528,\n",
              " 'Ġsmoking': 9216,\n",
              " 'Ġfringe': 26821,\n",
              " 'ĠEmb': 13302,\n",
              " 'ĠOman': 41782,\n",
              " 'ĠTraps': 44110,\n",
              " 'ikuman': 42889,\n",
              " 'Ġcreated': 2727,\n",
              " 'ĠSixers': 46321,\n",
              " 'eem': 13761,\n",
              " 'gram': 4546,\n",
              " 'Ġcert': 5051,\n",
              " 'Ġrebel': 14034,\n",
              " 'ĠTheresa': 20251,\n",
              " 'ĠInfinity': 22385,\n",
              " 'lesi': 36027,\n",
              " 'ĠPresents': 47102,\n",
              " 'imil': 26641,\n",
              " 'ĠCK': 45233,\n",
              " 'ĠCreep': 41682,\n",
              " 'Ġgive': 1577,\n",
              " 'Ġpleasure': 9476,\n",
              " 'ĠPopular': 22623,\n",
              " 'ĠNPCs': 28167,\n",
              " 'Ġoperative': 28609,\n",
              " 'Ġpavement': 31726,\n",
              " 'ĠTasman': 29958,\n",
              " 'Ġ196': 28817,\n",
              " 'ĠParticipation': 34408,\n",
              " 'Ã¦': 21241,\n",
              " 'edar': 44226,\n",
              " 'Ġviability': 32141,\n",
              " '¯': 107,\n",
              " 'Ġsurpassed': 30612,\n",
              " '688': 34427,\n",
              " 'Ġgood': 922,\n",
              " 'Ġaph': 49812,\n",
              " 'ĠVikings': 16420,\n",
              " '994': 42691,\n",
              " 'ĠAlf': 17136,\n",
              " 'Ġcomplaining': 18705,\n",
              " 'ĠGrand': 5675,\n",
              " 'even': 10197,\n",
              " 'ĠCube': 23315,\n",
              " 'mL': 32087,\n",
              " 'Ġcenturies': 10675,\n",
              " 're': 260,\n",
              " 'Ġ1906': 40538,\n",
              " 'bay': 24406,\n",
              " 'nesium': 27619,\n",
              " 'induced': 17223,\n",
              " 'ĠMartian': 32508,\n",
              " 'ĠStim': 41669,\n",
              " 'ĠSicily': 49301,\n",
              " 'Ġcorrid': 16080,\n",
              " 'Ġincrement': 18703,\n",
              " 'ĠSpeaker': 14931,\n",
              " 'stant': 18797,\n",
              " 'enza': 23674,\n",
              " 'Private': 29067,\n",
              " 'Ġdetach': 48224,\n",
              " 'Ġcustody': 10804,\n",
              " 'Ġavenues': 34265,\n",
              " 'ãĤ½': 47559,\n",
              " 'Ġcounty': 7968,\n",
              " 'Ġ1860': 37637,\n",
              " 'operated': 42767,\n",
              " 'Ġold': 1468,\n",
              " 'Ġsides': 5389,\n",
              " 'Ġformally': 15832,\n",
              " 'Ġwielded': 42459,\n",
              " 'ADE': 19266,\n",
              " 'ĠBry': 9092,\n",
              " 'Ġcustoms': 17112,\n",
              " 'ĠCynthia': 40897,\n",
              " ...}"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jMBijK00WtWn"
      },
      "source": [
        "### Test of different tokenizer\n",
        "\n",
        "Mögliche Modelle:\n",
        "* https://huggingface.co/models?pipeline_tag=text-classification&sort=downloads"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Aufgabe: Suche ein deutsches Modell und vergleiche das Ergebnis der Tokenisierung"
      ],
      "metadata": {
        "id": "hR79XPJa1Y4Z"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "BlqAZpP7AztE"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}