generate_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# W266 Final Project\n",
    "## Chapter Perspective Classification of Game of Thrones Text Passages\n",
    "### Fall 2019\n",
    "### T. P. Goter\n",
    "\n",
    "## Data Cleaning and Text Labeling\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import os\n",
    "from glob import glob\n",
    "import spacy\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "from matplotlib import pyplot as plt\n",
    "from utils import tokenization\n",
    "\n",
    "# Load in the english spacy model\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "line_sep = \"=\"*50 + \"\\n\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create the Dataset\n",
    "This portion of the code basically brings in five text files, one for each Game of Thrones book. It loops over each line in each book and assesses whether the line is a chapter title. In Game of Thrones, chapter titles are character names. In each of the text files, chapter titles are presented as the only word on a new line and in all capital letters. Therefore, with some simple string operations we can generate a set of chapter labels. In all there are 17 unique chapter titles, including a \"PROLOGUE\" title."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================================\n",
      "Processing the following files:\n",
      "./Data/raw_data/got_book1\n",
      "./Data/raw_data/got_book2\n",
      "./Data/raw_data/got_book3\n",
      "./Data/raw_data/got_book4\n",
      "./Data/raw_data/got_book5\n",
      "==================================================\n",
      "Processing ./Data/raw_data/got_book1\n",
      "HOUSE BARATHEON\n",
      "\n",
      "==================================================\n",
      "Processing ./Data/raw_data/got_book2\n",
      "APPENDIX—THE KINGS AND THEIR COURTS\n",
      "\n",
      "==================================================\n",
      "Processing ./Data/raw_data/got_book3\n",
      "Appendix\n",
      "\n",
      "==================================================\n",
      "Processing ./Data/raw_data/got_book4\n",
      "MEANWHILE, BACK ON THE WALL …\n",
      "\n",
      "==================================================\n",
      "Processing ./Data/raw_data/got_book5\n",
      "WESTEROS\n",
      "\n",
      "Generated 20 unique chapter titles\n",
      "The chapter titles are:\n",
      "ALAYNE\n",
      "ARYA\n",
      "BRAN\n",
      "BRIENNE\n",
      "CATELYN\n",
      "CERSEI\n",
      "DAENERYS\n",
      "DAVOS\n",
      "EDDARD\n",
      "EPILOGUE\n",
      "JAIME\n",
      "JON\n",
      "MELISANDRE\n",
      "PROLOGUE\n",
      "REEK\n",
      "SAMWELL\n",
      "SANSA\n",
      "THEON\n",
      "TYRION\n",
      "VICTARION\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "data_path = './Data/raw_data/'\n",
    "\n",
    "# \n",
    "book_files = sorted([data_path + book for book in os.listdir(data_path) if \"got_book\" in book])\n",
    "\n",
    "print(line_sep + \"Processing the following files:\")\n",
    "for book in book_files:\n",
    "    print(book)\n",
    "\n",
    "# Instantiate some empty objects for use in the loops below\n",
    "titles = set()\n",
    "titles_book = set()\n",
    "books = set()\n",
    "chapter_count = 0\n",
    "dataset = []\n",
    "chapter = \"\"\n",
    "\n",
    "book_ends = ['HOUSE BARATHEON\\n', \n",
    "            'APPENDIX—THE KINGS AND THEIR COURTS\\n',\n",
    "            'Appendix\\n',\n",
    "            'MEANWHILE, BACK ON THE WALL …\\n',\n",
    "            'WESTEROS\\n']\n",
    "\n",
    "# Loop over each book\n",
    "for book in book_files:\n",
    "    flag = False\n",
    "    print(line_sep + \"Processing {}\".format(book))\n",
    "    \n",
    "    # Open the book and read all of the lines in\n",
    "    with open(book, \"r\") as file_in:\n",
    "        book_lines = file_in.readlines()\n",
    "    \n",
    "    # Loop over every line in the current book\n",
    "    for line in book_lines:\n",
    "        \n",
    "        # Don't process anything beyond the last chapter\n",
    "        if line in book_ends:\n",
    "            print(line)\n",
    "            if line == book_ends[-1]:\n",
    "                dataset.append(chapter)\n",
    "            flag = False\n",
    "            break\n",
    "        \n",
    "        # Split the line into words, if applicable\n",
    "        words = line.split()\n",
    "        if len(words) == 1:\n",
    "            \n",
    "            # Chapter titles are just capitalized, multicharacter, names\n",
    "            if words[0].isupper() and words[0].isalpha() and len(words[0])>1:\n",
    "                               \n",
    "                # Maintain a set of chapter titles and a set of chapter titles with the book number\n",
    "                titles_book.add((words[0],book[-1]))\n",
    "                titles.add(words[0])\n",
    "                books.add(book[-1])\n",
    "                \n",
    "                # If we have found a chapter name, update the chapter number\n",
    "                chapter_count += 1\n",
    "                \n",
    "                if flag:\n",
    "                    # Dataset is a list of strings where each string is the entire text from the chapter\n",
    "                    dataset.append((chapter, book[-1]))\n",
    "                \n",
    "                # Reset the chapter string\n",
    "                chapter = \"\"\n",
    "                \n",
    "                # Flag to indicate we are in the text of the book\n",
    "                flag = True\n",
    "        \n",
    "        # Add the current line to the chapter string\n",
    "        if flag:\n",
    "            chapter += line  \n",
    "\n",
    "\n",
    "print(\"Generated {} unique chapter titles\".format(len(titles)))\n",
    "print(\"The chapter titles are:\")\n",
    "for t in sorted(titles):\n",
    "    print(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert Chapter Data to Sentence Data\n",
    "In the end we will want to make predictions on short groupings of sentences, not full chapters and not single sentences. But to start lets break the labeled chapters into labeled sentences using the sentencizer from spacy.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We will add tuples of sentence, label to a new list\n",
    "sentence_ds = []\n",
    "\n",
    "# Create a spacy pipeline that tokenizes and sentencizes our chapters.\n",
    "sentencizer = nlp.create_pipe(\"sentencizer\")\n",
    "nlp.add_pipe(sentencizer)\n",
    "\n",
    "# Create counter for labels\n",
    "d = 0\n",
    "\n",
    "# Loop over all the chapter data\n",
    "for doc in nlp.pipe([chap[0] for chap in dataset if chap[0].split()[0] in titles],\n",
    "                    disable=[\"tagger\", \"parser\", \"ner\", \"textcat\"]):\n",
    "    \n",
    "    # Loop over each sentence\n",
    "    for sent in doc.sents:\n",
    "        words = sent.text.split()\n",
    "        \n",
    "        # Remove chapter label from the first sentence in each chapter\n",
    "        if len(words) > 0 and words[0] in titles:\n",
    "            label = dataset[d][1]\n",
    "            new_sentence = \" \".join(words[1:])\n",
    "        else:\n",
    "            new_sentence = sent.text\n",
    "            \n",
    "        # Add sentence and label to the list\n",
    "        sentence_ds.append((new_sentence, label))\n",
    "    d +=1 \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean up\n",
    "Remove extraneous lines from our data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove dialogue quotes\n",
    "for ds in sentence_ds:   \n",
    "    # Get rid of lines that have Table of Contents in it as these are not part of the text\n",
    "    if 'Table of Contents' in ds[0]:\n",
    "        sentence_ds.remove(ds)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a DataFrame\n",
    "Let's push our information into a pandas dataframe with column labels. In doing this and exploring the data, it is uncovered that some extraneous page numbers have been treated as sentence beginnings. We remove these. We also remove blank lines. When we are done we reset our index and pickle our sentence/label dataframe for future use."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Push the clean dataset into a dataframe\n",
    "df = pd.DataFrame(sentence_ds, columns=['text', 'label'])\n",
    "\n",
    "# Place some marker in the text column if it was an empty line\n",
    "df.text = df.text.map(lambda x: 'REMOVE' if len(x.split()) < 1 else x)\n",
    "\n",
    "# Remove the zero-word sentences\n",
    "df = df[~df.text.str.match('^REMOVE')]\n",
    "\n",
    "# Remove leading numbers (likely page numbers introduced from the original copy/paste)\n",
    "df.text = df.text.str.replace('^[0-9]+\\s', '')\n",
    "df.text = df.text.str.replace('[”“]+', '')\n",
    "df.text = df.text.str.replace('[\\n]+', ' ')\n",
    "\n",
    "# Reset dataframe index so rows are sequentially indexed\n",
    "df = df.reset_index(drop=True)\n",
    "\n",
    "# Write dataframe to pickle file for later use\n",
    "df.to_pickle('got_sent_w_labels.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple Data Exploration\n",
    "We look at how many different sentences we have and their average size. We will combine sentences up to a maximum length of 200 below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================================\n",
      "There are 151196 total sentences in our dataset.\n",
      "==================================================\n",
      "Median sentence length of 9.0 words\n",
      "==================================================\n",
      "Mean sentence length of 11.3 words\n",
      "==================================================\n",
      "Minimum sentence length of 1 words\n",
      " Wind.\n",
      "==================================================\n",
      "Maximum sentence length of 168 words\n",
      "All of it came pouring out of Brienne then, like black blood from a wound; the betrayals and betrothals, Red Ronnet and his rose, Lord Renly dancing with her, the wager for her maidenhead, the bitter tears she shed the night her king wed Margaery Tyrell, the mêlée at Bitterbridge, the rainbow cloak that she had been so proud of, the shadow in the king’s pavilion, Renly dying in her arms, Riverrun and Lady Catelyn, the voyage down the Trident, dueling Jaime in the woods, the Bloody Mummers, Jaime crying Sapphires, Jaime in the tub at Harrenhal with steam rising from his body, the taste of Vargo Hoat’s blood when she bit down on his ear, the bear pit, Jaime leaping down onto the sand, the long ride to King’s Landing, Sansa Stark, the vow she’d sworn to Jaime, the vow she’d sworn to Lady Catelyn, Oathkeeper, Duskendale, Maidenpool, Nimble Dick and Crackclaw and the Whispers, the men she’d killed … I have to find her, she finished. \n"
     ]
    }
   ],
   "source": [
    "# Add sentence length to dataframe\n",
    "df['sent_length'] = df.text.map(lambda x: len(x.split()))\n",
    "\n",
    "#\n",
    "print(line_sep+\"There are {} total sentences in our dataset.\".format(len(df)))\n",
    "print(line_sep+\"Median sentence length of {} words\".format(df.sent_length.median()))\n",
    "print(line_sep+\"Mean sentence length of {:.1f} words\".format(df.sent_length.mean()))\n",
    "print(line_sep+\"Minimum sentence length of {} words\".format(df.sent_length.min()))\n",
    "print(df.iloc[df.sent_length.idxmin()].text)\n",
    "print(line_sep+\"Maximum sentence length of {} words\".format(df.sent_length.max()))\n",
    "print(df.iloc[df.sent_length.idxmax()].text)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create sequences\n",
    "Each sequence will be at least one sentence, but could be more. We use a word delimiter that results in TOKEN length sequences on the order of ~128 tokens in order to minimize the amount of wasted data. Based on the documentation for UDA, the 128 sequence length is the largest that can fit on an 11GB GPU (e.g., Colab)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "sents = list(df.text)\n",
    "x = 0 \n",
    "c = -1\n",
    "grouped_sent = \"\"\n",
    "groups = []\n",
    "while sents:\n",
    "    while x < 80: \n",
    "        if (c > 0 ) and len(grouped_sent) != 0 and ((df.loc[c-1].label != df.loc[c].label) or not sents):\n",
    "            break\n",
    "        grouped_sent += \" \" + sents.pop(0)\n",
    "        x = len(grouped_sent.split())\n",
    "        c += 1\n",
    "    groups.append((grouped_sent, df.loc[c].label))\n",
    "    grouped_sent = \"\"\n",
    "    x=0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create dataframe of examples\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_df = pd.DataFrame(groups,columns=['text', 'label'])\n",
    "\n",
    "# Add sequence length to dataframe\n",
    "group_df['seq_length'] = group_df.text.map(lambda x: len(x.split()))\n",
    "\n",
    "# Tokenize and add length of sequences that will be used by the actual model\n",
    "tokenizer = tokenization.FullTokenizer('bert_pretrained/bert_base/vocab.txt')\n",
    "group_df['tokens'] = group_df.text.map(lambda x: tokenizer.tokenize(x))\n",
    "group_df['token_length'] = group_df.tokens.map(lambda x: len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================================\n",
      "There are 19438 total sequences in our dataset.\n",
      "==================================================\n",
      "Median sequence length of 86.0 words\n",
      "==================================================\n",
      "Mean sequence length of 88.1 words\n",
      "==================================================\n",
      "Minimum sequence length of 54 words\n",
      " yet now were heard again, ringing from the timbers of her father’s hall: The King in the North!  The King in the North!  THE KING IN THE NORTH! The comet’s tail spread across the dawn, a red slash that bled above the crags of Dragonstone like a wound in the pink and purple sky.\n",
      "==================================================\n",
      "Maximum sequence length of 209 words\n",
      " Galladon drowned when I was four and he was eight, though, and Alysanne and Arianne died still in the cradle. I am the only child the gods let him keep. The freakish one, not fit to be a son or daughter. All of it came pouring out of Brienne then, like black blood from a wound; the betrayals and betrothals, Red Ronnet and his rose, Lord Renly dancing with her, the wager for her maidenhead, the bitter tears she shed the night her king wed Margaery Tyrell, the mêlée at Bitterbridge, the rainbow cloak that she had been so proud of, the shadow in the king’s pavilion, Renly dying in her arms, Riverrun and Lady Catelyn, the voyage down the Trident, dueling Jaime in the woods, the Bloody Mummers, Jaime crying Sapphires, Jaime in the tub at Harrenhal with steam rising from his body, the taste of Vargo Hoat’s blood when she bit down on his ear, the bear pit, Jaime leaping down onto the sand, the long ride to King’s Landing, Sansa Stark, the vow she’d sworn to Jaime, the vow she’d sworn to Lady Catelyn, Oathkeeper, Duskendale, Maidenpool, Nimble Dick and Crackclaw and the Whispers, the men she’d killed … I have to find her, she finished. \n",
      "==================================================\n",
      "\n",
      "==================================================\n",
      "Median sequence length of 113.0 TOKENS\n",
      "==================================================\n",
      "Mean sequence length of 114.7 TOKENS\n",
      "==================================================\n",
      "Minimum sequence length of 67 TOKENS\n",
      "==================================================\n",
      "Maximum sequence length of 298 TOKENS\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "print(line_sep+\"There are {} total sequences in our dataset.\".format(len(group_df)))\n",
    "print(line_sep+\"Median sequence length of {} words\".format(group_df.seq_length.median()))\n",
    "print(line_sep+\"Mean sequence length of {:.1f} words\".format(group_df.seq_length.mean()))\n",
    "print(line_sep+\"Minimum sequence length of {} words\".format(group_df.seq_length.min()))\n",
    "print(group_df.iloc[group_df.seq_length.idxmin()].text)\n",
    "print(line_sep+\"Maximum sequence length of {} words\".format(group_df.seq_length.max()))\n",
    "print(group_df.iloc[group_df.seq_length.idxmax()].text)\n",
    "print(line_sep)\n",
    "print(line_sep+\"Median sequence length of {} TOKENS\".format(group_df.token_length.median()))\n",
    "print(line_sep+\"Mean sequence length of {:.1f} TOKENS\".format(group_df.token_length.mean()))\n",
    "print(line_sep+\"Minimum sequence length of {} TOKENS\".format(group_df.token_length.min()))\n",
    "print(line_sep+\"Maximum sequence length of {} TOKENS\".format(group_df.token_length.max()))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Shuffle the Data\n",
    "Resample our dataframe to mix up the order."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19438\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>seq_length</th>\n",
       "      <th>tokens</th>\n",
       "      <th>token_length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>When the snows drifted forty and fifty feet h...</td>\n",
       "      <td>2</td>\n",
       "      <td>84</td>\n",
       "      <td>[when, the, snow, ##s, drifted, forty, and, fi...</td>\n",
       "      <td>104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>No telling how long he’s been here.  By the t...</td>\n",
       "      <td>3</td>\n",
       "      <td>80</td>\n",
       "      <td>[no, telling, how, long, he, ’, s, been, here,...</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>She could hear the people screaming at her, ...</td>\n",
       "      <td>2</td>\n",
       "      <td>85</td>\n",
       "      <td>[she, could, hear, the, people, screaming, at,...</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Stannis only seemed to half hear him.  I hav...</td>\n",
       "      <td>2</td>\n",
       "      <td>80</td>\n",
       "      <td>[stan, ##nis, only, seemed, to, half, hear, hi...</td>\n",
       "      <td>108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>She would sooner sit bathed in the ruddy glow...</td>\n",
       "      <td>5</td>\n",
       "      <td>94</td>\n",
       "      <td>[she, would, sooner, sit, bathed, in, the, rud...</td>\n",
       "      <td>121</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text label  seq_length  \\\n",
       "0   When the snows drifted forty and fifty feet h...     2          84   \n",
       "1   No telling how long he’s been here.  By the t...     3          80   \n",
       "2    She could hear the people screaming at her, ...     2          85   \n",
       "3    Stannis only seemed to half hear him.  I hav...     2          80   \n",
       "4   She would sooner sit bathed in the ruddy glow...     5          94   \n",
       "\n",
       "                                              tokens  token_length  \n",
       "0  [when, the, snow, ##s, drifted, forty, and, fi...           104  \n",
       "1  [no, telling, how, long, he, ’, s, been, here,...           100  \n",
       "2  [she, could, hear, the, people, screaming, at,...           102  \n",
       "3  [stan, ##nis, only, seemed, to, half, hear, hi...           108  \n",
       "4  [she, would, sooner, sit, bathed, in, the, rud...           121  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group_df = group_df.sample(frac=1.0).reset_index(drop=True)\n",
    "print(len(group_df))\n",
    "group_df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Separate into test, development, and training data\n",
    "\n",
    "There are 19,438 different pieces of data. We will use 15000 as training data, 2500 as development data and the rest as test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "Unknown property c",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-8b86e16ed492>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mgroup_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseq_length\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'tab:blue'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/pandas/plotting/_core.py\u001b[0m in \u001b[0;36mhist_series\u001b[0;34m(self, by, ax, grid, xlabelsize, xrot, ylabelsize, yrot, figsize, bins, **kwds)\u001b[0m\n\u001b[1;32m   2475\u001b[0m         \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2476\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2477\u001b[0;31m         \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2478\u001b[0m         \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2479\u001b[0m         \u001b[0maxes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/__init__.py\u001b[0m in \u001b[0;36minner\u001b[0;34m(ax, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1808\u001b[0m                         \u001b[0;34m\"the Matplotlib list!)\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlabel_namer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1809\u001b[0m                         RuntimeWarning, stacklevel=2)\n\u001b[0;32m-> 1810\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1811\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1812\u001b[0m         inner.__doc__ = _add_data_doc(inner.__doc__,\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/axes/_axes.py\u001b[0m in \u001b[0;36mhist\u001b[0;34m(self, x, bins, range, density, weights, cumulative, bottom, histtype, align, orientation, rwidth, log, color, label, stacked, normed, **kwargs)\u001b[0m\n\u001b[1;32m   6779\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpatch\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6780\u001b[0m                 \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 6781\u001b[0;31m                 \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   6782\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mlbl\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6783\u001b[0m                     \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_label\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlbl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/artist.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self, props)\u001b[0m\n\u001b[1;32m    914\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    915\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mcbook\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_setattr_cm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meventson\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 916\u001b[0;31m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0m_update_property\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    917\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    918\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mret\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/artist.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    914\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    915\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mcbook\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_setattr_cm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meventson\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 916\u001b[0;31m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0m_update_property\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    917\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    918\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mret\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/artist.py\u001b[0m in \u001b[0;36m_update_property\u001b[0;34m(self, k, v)\u001b[0m\n\u001b[1;32m    910\u001b[0m                 \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'set_'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    911\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 912\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Unknown property %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    913\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    914\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: Unknown property c"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEJpJREFUeJzt3X+s3XV9x/HnSyr4a7MgxbG22cVZF/GPAeuwm+4XOH4ayx+SdDGzcyxNDFvUbNMykxF/kBRdxJn4I0TYqjMiQzcaYdEOYWZ/8KP8qpTKWgWhwmxNAXVGtPreH+dTOZZ7e+8tt+cc+Dwfycn5ft/fz7nnfb73nvu631/npqqQJPXnOeNuQJI0HgaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVOLxt3AwRx77LE1NTU17jYk6Rnl9ttv/25VLZlt3EQHwNTUFFu2bBl3G5L0jJLkW3MZ5y4gSeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnq1ERfCazpTa2/btr6AxvOHXEnkp7J3AKQpE4ZAJLUKQNAkjrlMYBnkZmODYDHByQ9lVsAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnZpzACQ5IsmdSb7Y5k9IckuSHUk+l+TIVj+qze9sy6eGvsZFrX5fkjMX+sVIkuZuPlsAbwO2D81fClxWVSuAR4ELWv0C4NGqejlwWRtHkhOBNcCrgLOAjyU54um1L0k6VHMKgCTLgHOBT7b5AKcB17QhG4Hz2vTqNk9bfnobvxq4qqqeqKr7gZ3AqQvxIiRJ8zfXLYAPA+8EftbmXwI8VlX72vwuYGmbXgo8BNCWP97G/7w+zWMkSSM2awAkeT2wu6puHy5PM7RmWXawxww/37okW5Js2bNnz2ztSZIO0Vy2AF4DvCHJA8BVDHb9fBhYnGT/v5RcBjzcpncBywHa8hcDe4fr0zzm56rq8qpaWVUrlyxZMu8XJEmam1kDoKouqqplVTXF4CDuV6rqTcCNwBvbsLXAtW16U5unLf9KVVWrr2lnCZ0ArABuXbBXIkmal6fzT+HfBVyV5P3AncAVrX4F8OkkOxn85b8GoKq2JbkauBfYB1xYVT99Gs8vSXoa5hUAVXUTcFOb/ibTnMVTVT8Czp/h8ZcAl8y3SUnSwvNKYEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6tWjcDWhmU+uvG3cLkp7F3AKQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ2aNQCSPC/JrUnuTrItyXta/YQktyTZkeRzSY5s9aPa/M62fGroa13U6vclOfNwvShJ0uzmsgXwBHBaVf0mcBJwVpJVwKXAZVW1AngUuKCNvwB4tKpeDlzWxpHkRGAN8CrgLOBjSY5YyBcjSZq7WQOgBn7QZp/bbgWcBlzT6huB89r06jZPW356krT6VVX1RFXdD+wETl2QVyFJmrc5HQNIckSSu4DdwGbgG8BjVbWvDdkFLG3TS4GHANryx4GXDNeneczwc61LsiXJlj179sz/FUmS5mROAVBVP62qk4BlDP5qf+V0w9p9Zlg2U/3A57q8qlZW1colS5bMpT1J0iGY11lAVfUYcBOwClicZP//FF4GPNymdwHLAdryFwN7h+vTPEaSNGJzOQtoSZLFbfr5wOuA7cCNwBvbsLXAtW16U5unLf9KVVWrr2lnCZ0ArABuXagXIkman0WzD+F4YGM7Y+c5wNVV9cUk9wJXJXk/cCdwRRt/BfDpJDsZ/OW/BqCqtiW5GrgX2AdcWFU/XdiXI0maq1kDoKq2AidPU/8m05zFU1U/As6f4WtdAlwy/zYlSQvNK4ElqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUqUXjbkCjMbX+umnrD2w4d8SdSJoUbgFIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOjVrACRZnuTGJNuTbEvytlY/JsnmJDva/dGtniQfSbIzydYkpwx9rbVt/I4kaw/fy5IkzWYuWwD7gL+uqlcCq4ALk5wIrAduqKoVwA1tHuBsYEW7rQM+DoPAAC4GXg2cCly8PzQkSaM3awBU1SNVdUeb/j6wHVgKrAY2tmEbgfPa9GrgUzVwM7A4yfHAmcDmqtpbVY8Cm4GzFvTVSJLmbF7HAJJMAScDtwAvrapHYBASwHFt2FLgoaGH7Wq1meoHPse6JFuSbNmzZ8982pMkzcOcAyDJi4DPA2+vqu8dbOg0tTpI/RcLVZdX1cqqWrlkyZK5tidJmqc5BUCS5zL45f+ZqvpCK3+n7dqh3e9u9V3A8qGHLwMePkhdkjQGczkLKMAVwPaq+tDQok3A/jN51gLXDtXf3M4GWgU83nYRfQk4I8nR7eDvGa0mSRqDufw/gNcAfwp8LcldrfZ3wAbg6iQXAA8C57dl1wPnADuBHwJvAaiqvUneB9zWxr23qvYuyKuQJM3brAFQVf/N9PvvAU6fZnwBF87wta4ErpxPg5Kkw8MrgSWpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUqbl8GqiexabWXzdt/YEN5464E0mj5haAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnq1KJxN6DJNLX+umnrD2w4d8SdSDpc3AKQpE7NGgBJrkyyO8k9Q7VjkmxOsqPdH93qSfKRJDuTbE1yytBj1rbxO5KsPTwvR5I0V3PZAvhn4KwDauuBG6pqBXBDmwc4G1jRbuuAj8MgMICLgVcDpwIX7w8NSdJ4zBoAVfVVYO8B5dXAxja9EThvqP6pGrgZWJzkeOBMYHNV7a2qR4HNPDVUJEkjdKjHAF5aVY8AtPvjWn0p8NDQuF2tNlNdkjQmC30QONPU6iD1p36BZF2SLUm27NmzZ0GbkyQ96VAD4Dtt1w7tfner7wKWD41bBjx8kPpTVNXlVbWyqlYuWbLkENuTJM3mUANgE7D/TJ61wLVD9Te3s4FWAY+3XURfAs5IcnQ7+HtGq0mSxmTWC8GSfBb4Q+DYJLsYnM2zAbg6yQXAg8D5bfj1wDnATuCHwFsAqmpvkvcBt7Vx762qAw8sS5JGaNYAqKo/mWHR6dOMLeDCGb7OlcCV8+pOknTYeCWwJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmd8j+CaV78T2HSs4dbAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI65UdBaEH4ERHSM49bAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcrrAHRYeX2ANLncApCkThkAktQpA0CSOuUxAI2Fxwak8XMLQJI6ZQBIUqcMAEnqlMcANFE8NiCNjlsAktQpA0CSOjXyXUBJzgL+ETgC+GRVbRh1D3rmcdeQtPBGGgBJjgA+CvwxsAu4Lcmmqrp3lH3o2cNgkA7dqHcBnQrsrKpvVtWPgauA1SPuQZLE6HcBLQUeGprfBbx6xD2oAzNtGYyTWyWaNKMOgExTq18YkKwD1rXZHyS577B3NXAs8N0RPdehmOT+7G0OculTShPT2wwmuT97O7hfm8ugUQfALmD50Pwy4OHhAVV1OXD5KJsCSLKlqlaO+nnnapL7s7dDM8m9wWT3Z28LY9THAG4DViQ5IcmRwBpg04h7kCQx4i2AqtqX5C+BLzE4DfTKqto2yh4kSQMjvw6gqq4Hrh/1887ByHc7zdMk92dvh2aSe4PJ7s/eFkCqavZRkqRnHT8KQpI61W0AJFmc5JokX0+yPcnvJDkmyeYkO9r90WPq7R1JtiW5J8lnkzyvHTi/pfX2uXYQfVT9XJlkd5J7hmrTrqsMfCTJziRbk5wyht4+2L6vW5P8W5LFQ8suar3dl+TMUfc2tOxvklSSY9v82Ndbq/9VWzfbknxgqD6y9TZTf0lOSnJzkruSbElyaquPet0tT3Jj+72xLcnbWn0i3hPzUlVd3oCNwF+06SOBxcAHgPWtth64dAx9LQXuB57f5q8G/qzdr2m1TwBvHWFPvw+cAtwzVJt2XQHnAP/B4JqPVcAtY+jtDGBRm750qLcTgbuBo4ATgG8AR4yyt1ZfzuBEiG8Bx07Qevsj4D+Bo9r8ceNYbwfp78vA2UPr66YxrbvjgVPa9C8B/9PW0US8J+Zz63ILIMkvM/gBuwKgqn5cVY8x+FiKjW3YRuC88XTIIuD5SRYBLwAeAU4DrhlHb1X1VWDvAeWZ1tVq4FM1cDOwOMnxo+ytqr5cVfva7M0MrjfZ39tVVfVEVd0P7GTw8SQj6625DHgnv3gR5NjXG/BWYENVPdHG7B7qbWTr7SD9FfDLbfrFPHkN0ajX3SNVdUeb/j6wncEfbhPxnpiPLgMAeBmwB/inJHcm+WSSFwIvrapHYPBNBo4bdWNV9W3gH4AHGfzifxy4HXhs6JfaLgY/cOM007qa7uM+xtnrnzP46wsmoLckbwC+XVV3H7Bo7L0BrwB+r+1q/K8kvz1BvQG8HfhgkocYvEcuavWx9ZdkCjgZuIVnznvi53oNgEUMNi8/XlUnA//HYJNt7Np+w9UMNrV/FXghcPY0Qyf19K1ZP+5jVJK8G9gHfGZ/aZphI+styQuAdwN/P93iaWqjXm+LgKMZ7Kb4W+DqJGEyeoPBFso7qmo58A7aFjxj6i/Ji4DPA2+vqu8dbOg0tYl4//YaALuAXVV1S5u/hkEgfGf/plm73z3D4w+n1wH3V9WeqvoJ8AXgdxlsNu6/buMpH6ExBjOtq1k/7mMUkqwFXg+8qdqO2Ano7dcZBPvdSR5oz39Hkl+ZgN5oPXyh7aq4FfgZg8+1mYTeANYyeD8A/CtP7oYaeX9Jnsvgl/9nqmp/TxP9nphOlwFQVf8LPJTkN1rpdOBeBh9LsbbV1gLXjqG9B4FVSV7Q/vra39uNwBvH3NuwmdbVJuDN7cyHVcDj+zeLRyWDfzr0LuANVfXDoUWbgDVJjkpyArACuHVUfVXV16rquKqaqqopBr8YTmk/j2Nfb8C/MzjWRJJXMDg54ruMeb0NeRj4gzZ9GrCjTY903bX35RXA9qr60NCiiX1PzGjcR6HHdQNOArYAWxn84B8NvAS4gcEP1g3AMWPq7T3A14F7gE8zOPviZQzedDsZ/PVz1Aj7+SyD4xE/YfBL64KZ1hWDzd2PMjhT5GvAyjH0tpPBPte72u0TQ+Pf3Xq7j3ZGySh7O2D5Azx5FtAkrLcjgX9pP3d3AKeNY70dpL/XMjgedjeDfe6/NaZ191oGu3C2Dv2MnTMp74n53LwSWJI61eUuIEmSASBJ3TIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqf+Hy44gwMk1tvBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "group_df.seq_length.hist(bins=50, c='tab:blue')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['  Is it his fault the old man died? Stannis glanced into the fire.  I never wanted Cressen at that feast. He’d angered me, yes, he’d given me bad counsel, but I did not want him dead.  I’d hoped he might be granted a few years of ease and comfort. He had earned that much, at least, but—he ground his teeth together—but he died. And Pylos serves me ably.  Pylos is the least of it. The letter . . . What did your lords make of it, I wonder?']\n"
     ]
    }
   ],
   "source": [
    "a = group_df[group_df.text.str.contains(\"Stannis glanced into the fire\")].text\n",
    "print(a.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         92\n",
       "1         85\n",
       "2         94\n",
       "3         88\n",
       "4         86\n",
       "5         86\n",
       "6         85\n",
       "7         84\n",
       "8         88\n",
       "9         90\n",
       "10        82\n",
       "11        86\n",
       "12        86\n",
       "13        90\n",
       "14        81\n",
       "15        85\n",
       "16        90\n",
       "17        80\n",
       "18        89\n",
       "19        81\n",
       "20        99\n",
       "21        84\n",
       "22        83\n",
       "23        80\n",
       "24        93\n",
       "25        83\n",
       "26        88\n",
       "27        91\n",
       "28        88\n",
       "29        80\n",
       "        ... \n",
       "19408     89\n",
       "19409     84\n",
       "19410     85\n",
       "19411     81\n",
       "19412     86\n",
       "19413     83\n",
       "19414     88\n",
       "19415     83\n",
       "19416     85\n",
       "19417     82\n",
       "19418    107\n",
       "19419     82\n",
       "19420     93\n",
       "19421     84\n",
       "19422     97\n",
       "19423     87\n",
       "19424     90\n",
       "19425     83\n",
       "19426     86\n",
       "19427     82\n",
       "19428     90\n",
       "19429     90\n",
       "19430     84\n",
       "19431     92\n",
       "19432     80\n",
       "19433     97\n",
       "19434     86\n",
       "19435     82\n",
       "19436     93\n",
       "19437     89\n",
       "Name: seq_length, Length: 19438, dtype: int64"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group_df.seq_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "54"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group_df.seq_length.min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "209"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group_df.seq_length.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = group_df.loc[:15000]\n",
    "dev_df = group_df.loc[15000:17500]\n",
    "test_df = group_df.loc[17500:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.to_pickle('./Data/train/train.pkl')\n",
    "dev_df.to_pickle('./Data/dev/dev.pkl')\n",
    "test_df.to_pickle('./Data/test/test.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15001 2501 1938\n"
     ]
    }
   ],
   "source": [
    "print(len(train_df), len(dev_df), len(test_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGBdJREFUeJzt3X+QXeV93/H3JwJkmXWQKGYrC02lNHIn2JrIaCPUJm13wQEBnYjMxB0xjC3ZeJRmwLFdJY6IJ8E2phGxMVOmhESJVIvY9UbFpuyAXKwobBlmKhAiQj9QqNagwkqKFEdY9hpKLOfbP86z8mG5u/fu3dW5Bz+f18ydPec5zznne87uvZ97fty7igjMzCw/P9XpAszMrDMcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWabO6XQBE7noootiwYIFlazrBz/4Aeeff34l62pHnetzbe2rc32urT11qG337t3fiYh3Nu0YEbV9LF26NKry2GOPVbaudtS5PtfWvjrX59raU4fagKejhddYnwIyM8uUA8DMLFMOADOzTDUNAElvk/SUpGclHZD02dT+ZUkvStqTHktSuyTdI2lI0l5Jl5WWtVrSofRYffY2y8zMmmnlLqDXgSsiYkTSucATkr6Zpv12RDwwpv81wKL0uBy4D7hc0oXAbUAPEMBuSQMR8cp0bIiZmU1O0yOAdFF5JI2emx4T/ReZlcD9ab6dwGxJc4Grge0RcTK96G8HVkytfDMza1dL1wAkzZC0BzhB8SL+ZJp0RzrNc7ekmaltHvByafbh1DZeu5mZdYBiEv8SUtJs4EHgY8DfA38LnAdsBL4dEZ+T9AjwBxHxRJpnB/Ap4ApgZkR8PrX/HvBqRNw1Zh1rgbUA3d3dS/v7+6e2hS0aGRmhq6urknW1o871ubb21bk+19aeOtTW19e3OyJ6mnZs5cMC5QfFefzfGtPWCzychv8EuKE07XlgLnAD8Cel9jf0a/TwB8F+rM71ubb21bk+19aeOtRGix8Ea3oRWNI7gR9GxHclzQLeD9wpaW5EHJMk4Hpgf5plALhFUj/FReBTqd+jwH+SNCf1uwq4tWlC2ZssWP/IuNMOb7iuwkrM7K2slbuA5gJbJM2guGawNSIelvRXKRwE7AH+Q+q/DbgWGAJeBT4MEBEnJd0O7Er9PhcRJ6dvU8zMbDKaBkBE7AXe16D9inH6B3DzONM2A5snWaOZmZ0F/iSwmVmmav110DZ5410f8LUBMxvLRwBmZplyAJiZZcoBYGaWKQeAmVmmHABmZplyAJiZZcoBYGaWKQeAmVmmHABmZplyAJiZZcoBYGaWKQeAmVmmHABmZplyAJiZZcoBYGaWKQeAmVmmHABmZplyAJiZZcoBYGaWqaYBIOltkp6S9KykA5I+m9oXSnpS0iFJfyHpvNQ+M40PpekLSsu6NbU/L+nqs7VRZmbWXCtHAK8DV0TEzwNLgBWSlgN3AndHxCLgFeCm1P8m4JWI+Fng7tQPSZcCq4D3ACuAP5I0Yzo3xszMWtc0AKIwkkbPTY8ArgAeSO1bgOvT8Mo0Tpp+pSSl9v6IeD0iXgSGgGXTshVmZjZpiojmnYp36ruBnwXuBb4A7Ezv8pE0H/hmRLxX0n5gRUQMp2nfBi4HPpPm+Upq35TmeWDMutYCawG6u7uX9vf3T8d2NjUyMkJXV1cl62pHub59R05Nev7F8y6Y7pLOqPO+q3NtUO/6XFt76lBbX1/f7ojoadbvnFYWFhE/ApZImg08CPxco27pp8aZNl772HVtBDYC9PT0RG9vbyslTtng4CBVrasd5frWrH9k0vMfvrF3egsqqfO+q3NtUO/6XFt76lzbWJO6CygivgsMAsuB2ZJGA+QS4GgaHgbmA6TpFwAny+0N5jEzs4q1chfQO9M7fyTNAt4PHAQeA34tdVsNPJSGB9I4afpfRXGeaQBYle4SWggsAp6arg0xM7PJaeUU0FxgS7oO8FPA1oh4WNJzQL+kzwN/DWxK/TcBfy5piOKd/yqAiDggaSvwHHAauDmdWjIzsw5oGgARsRd4X4P2F2hwF09E/D/gA+Ms6w7gjsmXaWZm082fBDYzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsU638T2DrkAXrHzkzvG7xadaUxs3MpspHAGZmmXIAmJllqmkASJov6TFJByUdkPTx1P4ZSUck7UmPa0vz3CppSNLzkq4uta9IbUOS1p+dTTIzs1a0cg3gNLAuIp6R9A5gt6TtadrdEfHFcmdJlwKrgPcA7wL+UtK70+R7gV8GhoFdkgYi4rnp2BAzM5ucpgEQEceAY2n4+5IOAvMmmGUl0B8RrwMvShoClqVpQxHxAoCk/tTXAWBm1gGKiNY7SwuAx4H3Av8RWAN8D3ia4ijhFUn/BdgZEV9J82wCvpkWsSIiPpraPwhcHhG3jFnHWmAtQHd399L+/v52t21SRkZG6OrqqmRdrdp35NSZ4e5ZcPy19pe1eN4F01BRY3Xcd6PqXBvUuz7X1p461NbX17c7Inqa9Wv5NlBJXcDXgU9ExPck3QfcDkT6eRfwEUANZg8aX294U/pExEZgI0BPT0/09va2WuKUDA4OUtW6WrVmzG2gd+1r/67dwzf2TkNFjdVx342qc21Q7/pcW3vqXNtYLb2iSDqX4sX/qxHxDYCIOF6a/qfAw2l0GJhfmv0S4GgaHq/dzMwq1spdQAI2AQcj4kul9rmlbr8K7E/DA8AqSTMlLQQWAU8Bu4BFkhZKOo/iQvHA9GyGmZlNVitHAL8IfBDYJ2lPavtd4AZJSyhO4xwGfh0gIg5I2kpxcfc0cHNE/AhA0i3Ao8AMYHNEHJjGbTEzs0lo5S6gJ2h8Xn/bBPPcAdzRoH3bRPOZmVl1/ElgM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMNQ0ASfMlPSbpoKQDkj6e2i+UtF3SofRzTmqXpHskDUnaK+my0rJWp/6HJK0+e5tlZmbNtHIEcBpYFxE/BywHbpZ0KbAe2BERi4AdaRzgGmBReqwF7oMiMIDbgMuBZcBto6FhZmbVaxoAEXEsIp5Jw98HDgLzgJXAltRtC3B9Gl4J3B+FncBsSXOBq4HtEXEyIl4BtgMrpnVrzMysZYqI1jtLC4DHgfcCL0XE7NK0VyJijqSHgQ0R8URq3wH8DtALvC0iPp/afw94LSK+OGYdaymOHOju7l7a39/f9sZNxsjICF1dXZWsq1X7jpw6M9w9C46/1v6yFs+7YBoqaqyO+25UnWuDetfn2tpTh9r6+vp2R0RPs37ntLpASV3A14FPRMT3JI3btUFbTND+xoaIjcBGgJ6enujt7W21xCkZHBykqnW1as36R84Mr1t8mrv2tfzrepPDN/ZOQ0WN1XHfjapzbVDv+lxbe+pc21gt3QUk6VyKF/+vRsQ3UvPxdGqH9PNEah8G5pdmvwQ4OkG7mZl1QCt3AQnYBByMiC+VJg0Ao3fyrAYeKrV/KN0NtBw4FRHHgEeBqyTNSRd/r0ptZmbWAa2cU/hF4IPAPkl7UtvvAhuArZJuAl4CPpCmbQOuBYaAV4EPA0TESUm3A7tSv89FxMlp2QozM5u0pgGQLuaOd8L/ygb9A7h5nGVtBjZPpkAzMzs7/ElgM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLV/vcL21vKgtJXS5cd3nBdxZWYWV34CMDMLFMOADOzTDkAzMwy5QAwM8uUA8DMLFMOADOzTDkAzMwy5QAwM8uUA8DMLFNNA0DSZkknJO0vtX1G0hFJe9Lj2tK0WyUNSXpe0tWl9hWpbUjS+unfFDMzm4xWjgC+DKxo0H53RCxJj20Aki4FVgHvSfP8kaQZkmYA9wLXAJcCN6S+ZmbWIU2/CygiHpe0oMXlrQT6I+J14EVJQ8CyNG0oIl4AkNSf+j436YrNzGxaTOUawC2S9qZTRHNS2zzg5VKf4dQ2XruZmXWIIqJ5p+II4OGIeG8a7wa+AwRwOzA3Ij4i6V7gf0fEV1K/TcA2iqC5OiI+mto/CCyLiI81WNdaYC1Ad3f30v7+/qluY0tGRkbo6uqqZF2t2nfk1Jnh7llw/LXpX8fieRdMeRl13Hej6lwb1Ls+19aeOtTW19e3OyJ6mvVr6+ugI+L46LCkPwUeTqPDwPxS10uAo2l4vPaxy94IbATo6emJ3t7edkqctMHBQapaV6vWlL7Ced3i09y1b/q/vfvwjb1TXkYd992oOtcG9a7PtbWnzrWN1dYpIElzS6O/CozeITQArJI0U9JCYBHwFLALWCRpoaTzKC4UD7RftpmZTVXTt5SSvgb0AhdJGgZuA3olLaE4BXQY+HWAiDggaSvFxd3TwM0R8aO0nFuAR4EZwOaIODDtW2NmZi1r5S6gGxo0b5qg/x3AHQ3at1FcDzAzsxrwJ4HNzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMTf+Xy9hbyoLS9w2VHd5wXcWVmFnVfARgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllqmkASNos6YSk/aW2CyVtl3Qo/ZyT2iXpHklDkvZKuqw0z+rU/5Ck1Wdnc8zMrFWtHAF8GVgxpm09sCMiFgE70jjANcCi9FgL3AdFYAC3AZcDy4DbRkPDzMw6o2kARMTjwMkxzSuBLWl4C3B9qf3+KOwEZkuaC1wNbI+IkxHxCrCdN4eKmZlVqN1rAN0RcQwg/bw4tc8DXi71G05t47WbmVmHTPc/hFGDtpig/c0LkNZSnD6iu7ubwcHBaStuIiMjI5Wtq1XrFp8+M9w9643jZ9tk9kUd992oOtcG9a7PtbWnzrWN1W4AHJc0NyKOpVM8J1L7MDC/1O8S4Ghq7x3TPthowRGxEdgI0NPTE729vY26TbvBwUGqWler1pT+W9e6xae5a191/8Dt8I29Lfet474bVefaoN71ubb21Lm2sdo9BTQAjN7Jsxp4qNT+oXQ30HLgVDpF9ChwlaQ56eLvVanNzMw6pOlbSklfo3j3fpGkYYq7eTYAWyXdBLwEfCB13wZcCwwBrwIfBoiIk5JuB3alfp+LiLEXls3MrEJNAyAibhhn0pUN+gZw8zjL2QxsnlR1ZmZ21viTwGZmmXIAmJllygFgZpap6u4rtLeUBaVbUMsOb7iu4krM7GzxEYCZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYpB4CZWaYcAGZmmXIAmJllygFgZpYp/0MYm5RG/yhm3eLT9FZfiplN0ZSOACQdlrRP0h5JT6e2CyVtl3Qo/ZyT2iXpHklDkvZKumw6NsDMzNozHaeA+iJiSUT0pPH1wI6IWATsSOMA1wCL0mMtcN80rNvMzNp0Nq4BrAS2pOEtwPWl9vujsBOYLWnuWVi/mZm1YKoBEMC3JO2WtDa1dUfEMYD08+LUPg94uTTvcGozM7MOUES0P7P0rog4KuliYDvwMWAgImaX+rwSEXMkPQL8QUQ8kdp3AJ+KiN1jlrmW4hQR3d3dS/v7+9uubzJGRkbo6uqqZF2t2nfk1Jnh7llw/LUOFjOB7llw8YUXdLqMhur4ey2rc32urT11qK2vr2936bT8uKZ0F1BEHE0/T0h6EFgGHJc0NyKOpVM8J1L3YWB+afZLgKMNlrkR2AjQ09MTvb29UymxZYODg1S1rlatKd1xs27xae7aV8+bttYtPs2/r9m+G1XH32tZnetzbe2pc21jtX0KSNL5kt4xOgxcBewHBoDVqdtq4KE0PAB8KN0NtBw4NXqqyMzMqjeVt5TdwIOSRpfz3yLif0raBWyVdBPwEvCB1H8bcC0wBLwKfHgK6zYzsylqOwAi4gXg5xu0/z1wZYP2AG5ud31mZja9/FUQZmaZcgCYmWWqnreV2FtOo+8IAji84bqKKzGzVvkIwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlzwHYWeXPB5jVl48AzMwy5QAwM8uUA8DMLFMOADOzTPkisHWELw6bdZ6PAMzMMuUAMDPLlE8BWa341JBZdXwEYGaWKQeAmVmmKj8FJGkF8J+BGcCfRcSGqmuwtx6fGjKbfpUGgKQZwL3ALwPDwC5JAxHxXJV12E+O8YIBHA5mzVR9CmgZMBQRL0TEPwD9wMqKazAzM6o/BTQPeLk0PgxcXnENlokF6x9h3eLTrJngKKEV4x1JTHT0MZnlmHVK1QGgBm3xhg7SWmBtGh2R9PxZr6pwEfCditY1ab9Z4/p+0mvTndNTyzjLqe2+w7W1qw61/bNWOlUdAMPA/NL4JcDRcoeI2AhsrLIoAElPR0RP1ettVZ3rc23tq3N9rq09da5trKqvAewCFklaKOk8YBUwUHENZmZGxUcAEXFa0i3AoxS3gW6OiANV1mBmZoXKPwcQEduAbVWvtwWVn3aapDrX59raV+f6XFt76lzbGygimvcyM7OfOP4qCDOzTGUbAJJmS3pA0t9IOijpX0q6UNJ2SYfSzzkdqu2Tkg5I2i/pa5Leli6cP5lq+4t0Eb2qejZLOiFpf6mt4b5S4R5JQ5L2SrqsA7V9If1e90p6UNLs0rRbU23PS7q66tpK035LUki6KI13fL+l9o+lfXNA0h+W2ivbb+PVJ2mJpJ2S9kh6WtKy1F71vpsv6bH0unFA0sdTey2eE5MSEVk+gC3AR9PwecBs4A+B9altPXBnB+qaB7wIzErjW4E16eeq1PbHwG9UWNO/AS4D9pfaGu4r4FrgmxSf+VgOPNmB2q4CzknDd5ZquxR4FpgJLAS+DcyosrbUPp/iRoj/C1xUo/3WB/wlMDONX9yJ/TZBfd8Crintr8EO7bu5wGVp+B3A/0n7qBbPick8sjwCkPTTFH9gmwAi4h8i4rsUX0uxJXXbAlzfmQo5B5gl6Rzg7cAx4ArggU7UFhGPAyfHNI+3r1YC90dhJzBb0twqa4uIb0XE6TS6k+LzJqO19UfE6xHxIjBE8fUkldWW3A18ijd+CLLj+w34DWBDRLye+pwo1VbZfpugvgB+Og1fwI8/Q1T1vjsWEc+k4e8DByneuNXiOTEZWQYA8DPA3wH/VdJfS/ozSecD3RFxDIpfMnBx1YVFxBHgi8BLFC/8p4DdwHdLL2rDFH9wnTTevmr0dR+drPUjFO++oAa1SfoV4EhEPDtmUsdrA94N/Ot0qvF/SfqFGtUG8AngC5JepniO3JraO1afpAXA+4Anees8J87INQDOoTi8vC8i3gf8gOKQrePSecOVFIfa7wLOB65p0LWut281/bqPqkj6NHAa+OpoU4NuldUm6e3Ap4HfbzS5QVvV++0cYA7FaYrfBrZKEvWoDYojlE9GxHzgk6QjeDpUn6Qu4OvAJyLiexN1bdBWi+dvrgEwDAxHxJNp/AGKQDg+emiWfp4YZ/6z6f3AixHxdxHxQ+AbwL+iOGwc/dzGm75CowPG21dNv+6jCpJWA/8OuDHSidga1PbPKYL9WUmH0/qfkfRPa1AbqYZvpFMVTwH/SPG9NnWoDWA1xfMB4L/z49NQldcn6VyKF/+vRsRoTbV+TjSSZQBExN8CL0v6F6npSuA5iq+lWJ3aVgMPdaC8l4Dlkt6e3n2N1vYY8Gsdrq1svH01AHwo3fmwHDg1elhcFRX/dOh3gF+JiFdLkwaAVZJmSloILAKeqqquiNgXERdHxIKIWEDxwnBZ+nvs+H4D/gfFtSYkvZvi5ojv0OH9VnIU+Ldp+ArgUBqudN+l5+Um4GBEfKk0qbbPiXF1+ip0px7AEuBpYC/FH/4c4J8AOyj+sHYAF3aots8CfwPsB/6c4u6Ln6F40g1RvPuZWWE9X6O4HvFDihetm8bbVxSHu/dS3CmyD+jpQG1DFOdc96THH5f6fzrV9jzpjpIqaxsz/TA/vguoDvvtPOAr6e/uGeCKTuy3Cer7JYrrYc9SnHNf2qF990sUp3D2lv7Grq3Lc2IyD38S2MwsU1meAjIzMweAmVm2HABmZplyAJiZZcoBYGaWKQeAmVmmHABmZplyAJiZZer/A24IQDNojPyVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_df.seq_length.hist(bins=50)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAE3ZJREFUeJzt3X2sZPV93/H3J4Cxha0AwVxtllWXtFvJWDQYXWEkV9WtSXnyH2tLtgRCYXGQNmpBtaWt1HUqFScuEq6KrVpyiNZlZRw5xtQPYmVoyYZwZUUqjw7msZQbvDVrECgFY19bRVn32z/Ob53xMvdx772zu7/3SxrNme/5zczvfO/sfu45c2ZuqgpJUn9+bdITkCRNhgEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6tTJk57AYs4666zaunXrpKexIX72s59x2mmnTXoaE2cfBvZhYB8GK+3DY4899rdV9e6lxh3TAbB161YeffTRSU9jQ8zOzjIzMzPpaUycfRjYh4F9GKy0D0n+93LGeQhIkjplAEhSpwwASerUkgGQ5O1JHk7y/SRPJ/nDVj83yUNJnk/y9SRva/VT2+25tn7ryGN9qtWfS3LZem2UJGlpy9kDeBP4YFX9NnABcHmSi4HPAp+vqm3A68D1bfz1wOtV9Y+Az7dxJDkPuAp4L3A58MdJTlrLjZEkLd+SAVCD+XbzlHYp4IPAN1r9DuDDbXl7u01bf0mStPqdVfVmVf0AmAMuWpOtkCSt2LLeA0hyUpLHgVeB/cDfAD+uqkNtyEFgc1veDLwI0Na/AfzGaH3MfSRJG2xZnwOoql8AFyQ5Hfg28J5xw9p1Fli3UP1XJNkJ7ASYmppidnZ2OVM87s3Pz3ezrYuxDwP7MLAPg/Xqw4o+CFZVP04yC1wMnJ7k5PZb/jnAS23YQWALcDDJycCvA6+N1A8bvc/oc+wB9gBMT09XLx8C8QMvA/swsA8D+zBYrz4sGQBJ3g38XfvP/x3A7zC8sfsA8FHgTmAHcHe7y752+3+09X9ZVZVkH/BnST4H/CawDXh4jbenC1t337PgugO3fGgDZyLpeLacPYBNwB3tjJ1fA+6qqu8keQa4M8l/AP4auL2Nvx340yRzDL/5XwVQVU8nuQt4BjgE3NAOLUmSJmDJAKiqJ4D3jam/wJizeKrq/wIfW+CxbgZuXvk0JUlrzU8CS1KnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROregvgmljLfaHXyTpaLkHIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdWrJAEiyJckDSZ5N8nSST7T6p5P8KMnj7XLlyH0+lWQuyXNJLhupX95qc0l2r88mSZKWYzlfBncI2FVV30vyLuCxJPvbus9X1X8aHZzkPOAq4L3AbwJ/keQft9VfBP4FcBB4JMm+qnpmLTZEkrQySwZAVb0MvNyWf5rkWWDzInfZDtxZVW8CP0gyB1zU1s1V1QsASe5sYw0ASZqAFb0HkGQr8D7goVa6MckTSfYmOaPVNgMvjtztYKstVJckTcCy/x5AkncC3wQ+WVU/SXIb8Bmg2vWtwO8BGXP3YnzY1Jjn2QnsBJiammJ2dna5Uzyuzc/Pv2Vbd51/aMWPc7z3a1wfemQfBvZhsF59WFYAJDmF4T//r1bVtwCq6pWR9V8CvtNuHgS2jNz9HOCltrxQ/Zeqag+wB2B6erpmZmaWM8Xj3uzsLEdu63Wr+IMwB66ZWXLMsWxcH3pkHwb2YbBefVjOWUABbgeerarPjdQ3jQz7CPBUW94HXJXk1CTnAtuAh4FHgG1Jzk3yNoY3ivetzWZIklZqOXsAHwB+F3gyyeOt9gfA1UkuYDiMcwD4fYCqejrJXQxv7h4CbqiqXwAkuRG4DzgJ2FtVT6/htkiSVmA5ZwH9FeOP69+7yH1uBm4eU793sftJkjaOnwSWpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR16uRJT0Bra+vue8bWD9zyoQ2eiaRjnXsAktSpJQMgyZYkDyR5NsnTST7R6mcm2Z/k+XZ9RqsnyReSzCV5IsmFI4+1o41/PsmO9dssSdJSlrMHcAjYVVXvAS4GbkhyHrAbuL+qtgH3t9sAVwDb2mUncBsMgQHcBLwfuAi46XBoSJI23pIBUFUvV9X32vJPgWeBzcB24I427A7gw215O/CVGjwInJ5kE3AZsL+qXquq14H9wOVrujWSpGVb0ZvASbYC7wMeAqaq6mUYQiLJ2W3YZuDFkbsdbLWF6kc+x06GPQempqaYnZ1dyRSPW/Pz82/Z1l3nH1qzxz9e+jiuDz2yDwP7MFivPiw7AJK8E/gm8Mmq+kmSBYeOqdUi9V8tVO0B9gBMT0/XzMzMcqd4XJudneXIbb1ugTN6VuPANTNLjjkWjOtDj+zDwD4M1qsPyzoLKMkpDP/5f7WqvtXKr7RDO7TrV1v9ILBl5O7nAC8tUpckTcByzgIKcDvwbFV9bmTVPuDwmTw7gLtH6te2s4EuBt5oh4ruAy5NckZ78/fSVpMkTcByDgF9APhd4Mkkj7faHwC3AHcluR74IfCxtu5e4EpgDvg58HGAqnotyWeAR9q4P6qq19ZkKyRJK7ZkAFTVXzH++D3AJWPGF3DDAo+1F9i7kglKktaHnwSWpE4ZAJLUKQNAkjplAEhSp/w66E74NdGSjuQegCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnlgyAJHuTvJrkqZHap5P8KMnj7XLlyLpPJZlL8lySy0bql7faXJLda78pkqSVWM4ewJeBy8fUP19VF7TLvQBJzgOuAt7b7vPHSU5KchLwReAK4Dzg6jZWkjQhJy81oKq+m2TrMh9vO3BnVb0J/CDJHHBRWzdXVS8AJLmzjX1mxTOWJK2JJQNgETcmuRZ4FNhVVa8Dm4EHR8YcbDWAF4+ov3/cgybZCewEmJqaYnZ29iimePyYn59/y7buOv/Quj/vsdbfcX3okX0Y2IfBevVhtQFwG/AZoNr1rcDvARkzthh/qKnGPXBV7QH2AExPT9fMzMwqp3h8mZ2d5chtvW73Pev+vAeumVlyzEYa14ce2YeBfRisVx9WFQBV9crh5SRfAr7Tbh4EtowMPQd4qS0vVJckTcCqTgNNsmnk5keAw2cI7QOuSnJqknOBbcDDwCPAtiTnJnkbwxvF+1Y/bUnS0VpyDyDJ14AZ4KwkB4GbgJkkFzAcxjkA/D5AVT2d5C6GN3cPATdU1S/a49wI3AecBOytqqfXfGskScu2nLOArh5Tvn2R8TcDN4+p3wvcu6LZSZLWjZ8ElqROGQCS1CkDQJI6ZQBIUqeO5pPAWiNbd9/DrvMPbcgHvyTpMPcAJKlTBoAkdcpDQJ3busBhpwO3fGiDZyJpo7kHIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ1aMgCS7E3yapKnRmpnJtmf5Pl2fUarJ8kXkswleSLJhSP32dHGP59kx/psjiRpuZazB/Bl4PIjaruB+6tqG3B/uw1wBbCtXXYCt8EQGMBNwPuBi4CbDoeGJGkylgyAqvou8NoR5e3AHW35DuDDI/Wv1OBB4PQkm4DLgP1V9VpVvQ7s562hIknaQKt9D2Cqql4GaNdnt/pm4MWRcQdbbaG6JGlCTl7jx8uYWi1Sf+sDJDsZDh8xNTXF7Ozsmk3uWLXr/ENMvWO4PlZMqu/z8/Nd/MyXYh8G9mGwXn1YbQC8kmRTVb3cDvG82uoHgS0j484BXmr1mSPqs+MeuKr2AHsApqena2ZmZtywE8p1u+9h1/mHuPXJtc7j1TtwzcxEnnd2dpYefuZLsQ8D+zBYrz6s9hDQPuDwmTw7gLtH6te2s4EuBt5oh4juAy5NckZ78/fSVpMkTciSv3Im+RrDb+9nJTnIcDbPLcBdSa4Hfgh8rA2/F7gSmAN+DnwcoKpeS/IZ4JE27o+q6sg3liVJG2jJAKiqqxdYdcmYsQXcsMDj7AX2rmh2kqR14yeBJalTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdOnb+CG0Htu6+Z9JTkKRfcg9AkjplAEhSpzwEpLEWOlx14JYPbfBMJK0X9wAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSp44qAJIcSPJkkseTPNpqZybZn+T5dn1GqyfJF5LMJXkiyYVrsQGSpNVZiz2Af15VF1TVdLu9G7i/qrYB97fbAFcA29plJ3DbGjy3JGmV1uODYNuBmbZ8BzAL/NtW/0pVFfBgktOTbKqql9dhDlonfkBMOnEc7R5AAX+e5LEkO1tt6vB/6u367FbfDLw4ct+DrSZJmoCj3QP4QFW9lORsYH+S/7nI2Iyp1VsGDUGyE2BqaorZ2dmjnOKxY9f5hxZcN/WOxdcf69bq5zQ/P39C/cxXyz4M7MNgvfpwVAFQVS+161eTfBu4CHjl8KGdJJuAV9vwg8CWkbufA7w05jH3AHsApqena2Zm5mimeEy5bpGvg951/iFuffL4/WqmA9fMrMnjzM7OciL9zFfLPgzsw2C9+rDqQ0BJTkvyrsPLwKXAU8A+YEcbtgO4uy3vA65tZwNdDLzh8X9Jmpyj+ZVzCvh2ksOP82dV9d+TPALcleR64IfAx9r4e4ErgTng58DHj+K5JUlHadUBUFUvAL89pv5/gEvG1Au4YbXPJ0laW34SWJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdOn6/gF7HFP9UpHT8cQ9AkjplAEhSpwwASeqUASBJnTIAJKlTngWkdeXZQdKxyz0ASeqUASBJnfIQkCZioUNDX778tA2eidQv9wAkqVMGgCR1ykNA62ChwxuSdCxxD0CSOrXhewBJLgf+M3AS8F+q6paNnoOOXU/+6A2uG7MH5ecGpLW3oXsASU4CvghcAZwHXJ3kvI2cgyRpsNF7ABcBc1X1AkCSO4HtwDMbPA8dZ1bzvop7DdLiNjoANgMvjtw+CLx/g+ewZnyz99i20p+PgaHebHQAZEytfmVAshPY2W7OJ3lu3Wd1DPjXcBbwt5Oex6RNsg/57CSedUG+Hgb2YbDSPvyD5Qza6AA4CGwZuX0O8NLogKraA+zZyEkdC5I8WlXTk57HpNmHgX0Y2IfBevVho08DfQTYluTcJG8DrgL2bfAcJEls8B5AVR1KciNwH8NpoHur6umNnIMkabDhnwOoqnuBezf6eY8D3R32WoB9GNiHgX0YrEsfUlVLj5IknXD8KghJ6pQBsEGS7E3yapKnRmpnJtmf5Pl2fUarJ8kXkswleSLJhZOb+dpaoA+fTvKjJI+3y5Uj6z7V+vBckssmM+u1lWRLkgeSPJvk6SSfaPWuXg+L9KG318Pbkzyc5PutD3/Y6ucmeai9Hr7eTpwhyant9lxbv3XVT15VXjbgAvwz4ELgqZHafwR2t+XdwGfb8pXAf2P43MTFwEOTnv869+HTwL8ZM/Y84PvAqcC5wN8AJ016G9agB5uAC9vyu4D/1ba1q9fDIn3o7fUQ4J1t+RTgofZzvgu4qtX/BPiXbflfAX/Slq8Cvr7a53YPYINU1XeB144obwfuaMt3AB8eqX+lBg8CpyfZtDEzXV8L9GEh24E7q+rNqvoBMMfwdSLHtap6uaq+15Z/CjzL8Cn5rl4Pi/RhISfq66Gqar7dPKVdCvgg8I1WP/L1cPh18g3gkiTjPmS7JANgsqaq6mUY/jEAZ7f6uK/MWOwfxongxnZ4Y+/hQx900Ie2+/4+ht/6un09HNEH6Oz1kOSkJI8DrwL7GfZuflxVh9qQ0W39ZR/a+jeA31jN8xoAx6YlvzLjBHMb8A+BC4CXgVtb/YTuQ5J3At8EPllVP1ls6JjaidyH7l4PVfWLqrqA4dsRLgLeM25Yu16zPhgAk/XK4V35dv1qqy/5lRknkqp6pf0D+H/Al/j73foTtg9JTmH4T++rVfWtVu7u9TCuDz2+Hg6rqh8DswzvAZye5PBntUa39Zd9aOt/neUfVv0VBsBk7QN2tOUdwN0j9Wvb2R8XA28cPjRwIjriePZHgMNnCO0DrmpnPZwLbAMe3uj5rbV2vPZ24Nmq+tzIqq5eDwv1ocPXw7uTnN6W3wH8DsP7IQ8AH23Djnw9HH6dfBT4y2rvCK/YpN8B7+UCfI1hd/bvGBL8eobjdvcDz7frM+vvzwr4IsNxwCeB6UnPf5378KdtO59oL+5NI+P/XevDc8AVk57/GvXgnzLssj8BPN4uV/b2elikD729Hv4J8Ndte58C/n2r/xZDwM0B/xU4tdXf3m7PtfW/tdrn9pPAktQpDwFJUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOvX/AQLxod/4gAxgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_df.token_length.hist(bins=50)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "dummy = np.array([3,4,5])\n",
    "dummy_temp = np.array([3,4,5])/0.25\n",
    "sm_dummy = np.exp(dummy)/np.sum(np.exp(dummy))\n",
    "sm_dummy_temp = np.exp(dummy_temp)/np.sum(np.exp(dummy_temp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.09003057, 0.24472847, 0.66524096])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sm_dummy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3.29320439e-04, 1.79802867e-02, 9.81690393e-01])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sm_dummy_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}