Harry-Potter-Generation-RNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F \n",
    "import numpy as np\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# One-hot encoding matrix with dim (seq_length,K)\n",
    "# (creating one hot encoded rows for each character in a sequence )\n",
    "def one_hot(text_seq):\n",
    "    data = np.zeros((len(text_seq), K), dtype=np.float32)\n",
    "    for i in range(len(text_seq)):\n",
    "        char_as_int = text_seq[i]\n",
    "        data[i][char_as_int] = 1\n",
    "    return data\n",
    "\n",
    "def get_all_sequences(book_text):\n",
    "    # Translate text to integers\n",
    "    book_as_ints = np.array([c2i[c] for c in book_text])\n",
    "    n_seq = len(book_as_ints)//seq_length\n",
    "    book_text = book_text[:n_seq*seq_length]\n",
    "    batches = []\n",
    "    for i in range(0, len(book_text), seq_length):\n",
    "        batches.append(one_hot(book_as_ints[i:i+seq_length]))\n",
    "    return np.array(batches, dtype=np.float32)\n",
    "\n",
    "# one_hots = (seq_length, K)\n",
    "def one_hot_to_txt(one_hots):\n",
    "    chars = []\n",
    "    for arr in one_hots:\n",
    "        idx = np.where(arr == 1)[0][0]\n",
    "        chars.append(i2c[idx])\n",
    "    return ''.join(chars)\n",
    "\n",
    "def one_hot_to_ind(one_hots):\n",
    "    indices = []\n",
    "    for arr in one_hots:\n",
    "        indices.append(np.where(arr==1)[0][0])\n",
    "    return np.array(indices)\n",
    "\n",
    "def get_next_char(s):\n",
    "    s = nn.functional.softmax(s, dim=1)\n",
    "    l = torch.cumsum(s, dim=1)\n",
    "    a = random.uniform(0,1)\n",
    "    ls = (l - a).cpu().detach().numpy()\n",
    "    #print(ls)\n",
    "    idx = np.where(ls > 0)\n",
    "    idx = idx[1][0]\n",
    "    onehot = np.zeros(K, dtype=np.float32)\n",
    "    onehot[idx] = 1\n",
    "    onehot = torch.from_numpy(onehot).view(1,1,K).float().to(device)\n",
    "    return idx, i2c[idx], onehot\n",
    "\n",
    "def synthesize_text(model, char, length):\n",
    "    model.eval()\n",
    "    generated_seq = [char]\n",
    "    x0 = np.zeros(K, dtype=np.float32)\n",
    "    x0[c2i[char]] = 1\n",
    "    x0 = torch.from_numpy(x0)\n",
    "    x0 = x0.view(1,1,K).to(device)\n",
    "    hidden = model.init_hidden(1)\n",
    "    for i in range(length):\n",
    "        #print(\"x0 shape: \" + str(x0.shape))\n",
    "        #print(\"hidden shape: \" + str(hidden.shape))\n",
    "        s, hidden = model(x0, hidden)\n",
    "        i, c, x0 = get_next_char(s)\n",
    "        generated_seq.append(c)\n",
    "    model.train()\n",
    "    return ''.join(generated_seq)\n",
    "\n",
    "# Vanilla RNN for character-level generation\n",
    "class VanillaRNN(nn.Module):\n",
    "    def __init__(self, input_size, output_size, hidden_size, n_layers):\n",
    "        super().__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.n_layers = n_layers\n",
    "        self.rnn = nn.RNN(input_size, hidden_size, n_layers, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_size, len(chars))\n",
    "\n",
    "    def forward(self, x, hidden):\n",
    "        # Passing in the input and hidden state into the model and obtaining outputs\n",
    "        out, hidden = self.rnn(x, hidden)\n",
    "        # Reshaping the outputs such that it can be fit into the fully connected layer\n",
    "        out = out.contiguous().view(-1, self.hidden_size)\n",
    "        out = self.fc(out)\n",
    "\n",
    "        return out, hidden\n",
    "\n",
    "    def init_hidden(self, batch_size):\n",
    "        # This method generates the first hidden state of zeros which we'll use in the forward pass\n",
    "        # We'll send the tensor holding the hidden state to the device we specified earlier as well\n",
    "        if cuda_available:\n",
    "            hidden = torch.zeros(self.n_layers, batch_size, self.hidden_size).cuda()\n",
    "        else:\n",
    "            hidden = torch.zeros(self.n_layers, batch_size, self.hidden_size)\n",
    "        return hidden\n",
    "\n",
    "# Saves loss and generted text during training    \n",
    "lossarr = []\n",
    "strarr = []\n",
    "def train(model, n_epochs, clip, debug = 0, steps_during_debug=2):\n",
    "    model.train()\n",
    "    update_step = 0\n",
    "    smooth_loss = 0\n",
    "    debug = 0\n",
    "    #print(\"Numbers of update steps per epoch: \" + str(x.shape[0]))\n",
    "    for epoch in range(1, n_epochs + 1):\n",
    "        #OBSSS!!! HARDCODED BATCH MIGHT CHANGE LATER\n",
    "        hidden = model.init_hidden(1)\n",
    "        for i in range(x.shape[0]):\n",
    "            if debug: \n",
    "                if(i > steps_during_debug): \n",
    "                    break    \n",
    "            # Detatching hidden state from the computational graph \n",
    "            # (see truncated back propagation through time) such that\n",
    "            # the graph doesn't become super big, slowing down computations\n",
    "            # and exploding the gradient. \n",
    "            \n",
    "            # Note however that the hidden state VALUE will still be inherited\n",
    "            # We just say that it doesn't depend on previous hidden states\n",
    "            hidden.detach_()\n",
    "            hidden = hidden.to(device)\n",
    "            \n",
    "            # Clear gradients between update steps, won't be cleared automatically. \n",
    "            opt.zero_grad() \n",
    "            \n",
    "            #Remember that input should have dimensions (batch_size, seq_size, number_of_classes)\n",
    "            seq = x[i]\n",
    "            seq = torch.from_numpy(seq).view(1,seq_length,K)\n",
    "            #if cuda_available: \n",
    "            seq = seq.to(device)\n",
    "            #print(\"seq shape: \" + str(seq.shape))        \n",
    "\n",
    "            #Should have shape (25) (not (25,80))\n",
    "            target = y[i]\n",
    "            target = one_hot_to_ind(target)\n",
    "            target = torch.from_numpy(target)\n",
    "            target = target.to(device)\n",
    "            #print(\"target shape: \" + str(target.shape))\n",
    "\n",
    "            output, hidden = model(seq, hidden)\n",
    "            loss = criterion(output, target)\n",
    "            \n",
    "            if update_step == 0:\n",
    "                smooth_loss = loss\n",
    "            else:\n",
    "                smooth_loss = smooth_loss*0.999 + loss*0.001\n",
    "            if update_step%500 == 0:\n",
    "                lossarr.append(smooth_loss.item())\n",
    "                print(len(lossarr))\n",
    "                print(\"\\n\\n===========================================================================================\")\n",
    "                print(\"Update Step: \" + str(update_step) +\"/\" + str(x.shape[0]) + \", \" + str((update_step/x.shape[0])*100) +\"%\")\n",
    "                print(str(smooth_loss))\n",
    "                print(\"========================================= TEXT ============================================\")\n",
    "                print(synthesize_text(model, 'H', 50))\n",
    "                print(\"===========================================================================================\")\n",
    "            if update_step%20000 == 0:\n",
    "                strarr.append(synthesize_text(model, 'H', 50))\n",
    "            \n",
    "            # BPTT \n",
    "            loss.backward() \n",
    "            # Clip gradients such that they don't explode\n",
    "            nn.utils.clip_grad_norm_(model.parameters(), clip) \n",
    "            # Update the gradients\n",
    "            opt.step() \n",
    "            update_step += 1\n",
    "    model.eval()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on: cuda\n"
     ]
    }
   ],
   "source": [
    "cuda_available = torch.cuda.is_available()\n",
    "if cuda_available:\n",
    "    device = torch.device(\"cuda\")\n",
    "else:\n",
    "    device = torch.device(\"cpu\")\n",
    "\n",
    "print(\"Running on: \" + str(device))\n",
    "\n",
    "# Open file and get text\n",
    "file = open(\"Clean_All_Books.txt\", \"r\", encoding=\"utf8\")\n",
    "book = file.read()\n",
    "file.close()\n",
    "\n",
    "# Tokenize chars in text\n",
    "chars = list(set(book))\n",
    "chars.sort()\n",
    "i2c = dict(enumerate(chars))\n",
    "c2i = {i2c[i]: i for i in i2c.keys()}\n",
    "\n",
    "# Hyper-parameters\n",
    "# Num hidden nodes\n",
    "m = 100\n",
    "# Batch Size\n",
    "seq_length = 25\n",
    "# Number of chars\n",
    "K = len(chars)\n",
    "# Learning rate\n",
    "eta = 0.1\n",
    "      \n",
    "model = VanillaRNN(input_size = K, output_size = K, hidden_size = m, n_layers=1)\n",
    "model.to(device)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "opt = torch.optim.Adagrad(model.parameters(), lr=eta)\n",
    "# model.parameters are all params e.g U, V, b, c etc, see print below\n",
    "#for param in list(model.parameters()):\n",
    "#    print(param.shape)\n",
    "x = get_all_sequences(book)\n",
    "#print(np.shape(x))\n",
    "#print(one_hot_to_txt(x[0]))\n",
    "# If even number of chars (in relation to seq_length), first char will be last target\n",
    "y = get_all_sequences(book[1:] + book[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Numbers of update steps per epoch: 256485\n",
      "1\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 0/256485, 0.0%\n",
      "tensor(4.5091, device='cuda:0', grad_fn=<NllLossBackward>)\n",
      "========================================= TEXT ============================================\n",
      "Hk8XC2,LBu—~QVq:Qg4txj\\C%0j!Ov*welwR’\n",
      "R1:&?NG4R\"E,d\n",
      "===========================================================================================\n",
      "2\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 500/256485, 0.1949431740647601%\n",
      "tensor(3.9956, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H, . d a’o, ee toe opabe ad n fly \n",
      "uie cey, ageh n \n",
      "===========================================================================================\n",
      "3\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 1000/256485, 0.3898863481295202%\n",
      "tensor(3.4975, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hw hotipt’ore’selad fethes in Cro, Doothonithetunt \n",
      "===========================================================================================\n",
      "4\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 1500/256485, 0.5848295221942803%\n",
      "tensor(3.1111, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hen., ommufveod thamm sacon’d capumin’conb t Tord ’\n",
      "===========================================================================================\n",
      "5\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 2000/256485, 0.7797726962590404%\n",
      "tensor(2.8525, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harlked,,””” \n",
      "rediy goabll tars, is bis Dus \n",
      "\n",
      "fivde\n",
      "===========================================================================================\n",
      "6\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 2500/256485, 0.9747158703238007%\n",
      "tensor(2.6648, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HoTGawt atoutey licge arf bedpurlit \n",
      "ouccus starnaO\n",
      "===========================================================================================\n",
      "7\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 3000/256485, 1.1696590443885606%\n",
      "tensor(2.5507, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Had avard coud!ed Hasliy ferkabeoskey, svedled,” sh\n",
      "===========================================================================================\n",
      "8\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 3500/256485, 1.3646022184533209%\n",
      "tensor(2.4967, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har. souleneling \n",
      "lithe sew sor mliw’s he taghill S\n",
      "===========================================================================================\n",
      "9\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 4000/256485, 1.5595453925180809%\n",
      "tensor(2.4508, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H’t \n",
      "peot could Ha cave salit thereley, soatare, Sy\n",
      "===========================================================================================\n",
      "10\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 4500/256485, 1.754488566582841%\n",
      "tensor(2.3869, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Ha \n",
      "iy in erapad.” \n",
      "“Dund yeatd,. Ro the’ged \n",
      "f hin\n",
      "===========================================================================================\n",
      "11\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 5000/256485, 1.9494317406476014%\n",
      "tensor(2.3464, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He” in gleugm masing \n",
      "awhint Mutwer heel \n",
      "Iupeedtan\n",
      "===========================================================================================\n",
      "12\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 5500/256485, 2.1443749147123614%\n",
      "tensor(2.2916, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Ha wey’ the canw there the hamd hift \n",
      "noo there cus\n",
      "===========================================================================================\n",
      "13\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 6000/256485, 2.339318088777121%\n",
      "tensor(2.2666, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Horryye hon Hiars, \n",
      "thein thiptreyeyshee, out int \n",
      "\n",
      "===========================================================================================\n",
      "14\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 6500/256485, 2.534261262841882%\n",
      "tensor(2.2354, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harre to \n",
      "HarryKbed on shered of concor licks wof, \n",
      "===========================================================================================\n",
      "15\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 7000/256485, 2.7292044369066417%\n",
      "tensor(2.2287, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Haly, ald, pocle Hatrer a mpotry ass ar be sred”’ t\n",
      "===========================================================================================\n",
      "16\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 7500/256485, 2.924147610971402%\n",
      "tensor(2.2227, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hirryyies eore \n",
      "Whears a beuwn. one-feo he ber. \n",
      "Th\n",
      "===========================================================================================\n",
      "17\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 8000/256485, 3.1190907850361618%\n",
      "tensor(2.2005, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hat aid \n",
      "Wacly the \n",
      "Srond, on nepbont doa \n",
      "s ith a \n",
      "===========================================================================================\n",
      "18\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 8500/256485, 3.314033959100922%\n",
      "tensor(2.1792, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hored unlty dors floul frounonithe wes who the on h\n",
      "===========================================================================================\n",
      "19\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 9000/256485, 3.508977133165682%\n",
      "tensor(2.1526, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harver. IN Ccoung ver dicane beett the e lilf got a\n",
      "===========================================================================================\n",
      "20\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 9500/256485, 3.7039203072304425%\n",
      "tensor(2.1432, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hatet hey that ryanvery his fork, Beveed ou. \n",
      "“Woud\n",
      "===========================================================================================\n",
      "21\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 10000/256485, 3.8988634812952028%\n",
      "tensor(2.1272, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hars agmulldt bet you thot. \n",
      "En Wo.” \n",
      "“Yous. \n",
      "“Dull\n",
      "===========================================================================================\n",
      "22\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 10500/256485, 4.093806655359963%\n",
      "tensor(2.0862, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HO’rdaigh weararer haghe thee hermy \n",
      "aver, \n",
      "Bond we\n",
      "===========================================================================================\n",
      "23\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 11000/256485, 4.288749829424723%\n",
      "tensor(2.0873, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harray \n",
      "flockes us — woslus exow at tasd \n",
      "saster, t\n",
      "===========================================================================================\n",
      "24\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 11500/256485, 4.483693003489483%\n",
      "tensor(2.0800, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harrit hey none \n",
      "peelleste hes him, Harrexre was, \n",
      "\n",
      "===========================================================================================\n",
      "25\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 12000/256485, 4.678636177554242%\n",
      "tensor(2.0690, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harring, rowdore, it ang sullows \n",
      "the wos mad, the \n",
      "===========================================================================================\n",
      "26\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 12500/256485, 4.873579351619003%\n",
      "tensor(2.0397, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HT’rmye \n",
      "I Ned is sulb Harry hors Cfaile the sol’s \n",
      "===========================================================================================\n",
      "27\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 13000/256485, 5.068522525683764%\n",
      "tensor(2.0302, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Herly? “Sieveridly tene thit plorets on nothing has\n",
      "===========================================================================================\n",
      "28\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 13500/256485, 5.263465699748523%\n",
      "tensor(2.0364, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Haon’s hon in poml you whareashing serourw whey lat\n",
      "===========================================================================================\n",
      "29\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 14000/256485, 5.458408873813283%\n",
      "tensor(2.0193, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harys letrisoint \n",
      "botales ot \n",
      "hish was of teruid, l\n",
      "===========================================================================================\n",
      "30\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 14500/256485, 5.653352047878044%\n",
      "tensor(2.0110, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry the Bilehw ar \n",
      "geed puld ded to ked anke pak,\n",
      "===========================================================================================\n",
      "31\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 15000/256485, 5.848295221942804%\n",
      "tensor(2.0108, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "His withu wef saif noem wis ow \n",
      "pant there sich fac\n",
      "===========================================================================================\n",
      "32\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 15500/256485, 6.043238396007563%\n",
      "tensor(2.0039, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H....... shat ter often onder bee ougviss offy. \n",
      "“J\n",
      "===========================================================================================\n",
      "33\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 16000/256485, 6.2381815700723235%\n",
      "tensor(1.9771, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H.”’ erone on Ninnt hot “Learr hes thoug. It sturn \n",
      "===========================================================================================\n",
      "34\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 16500/256485, 6.433124744137084%\n",
      "tensor(1.9623, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Her ere tei hare \n",
      "eakd at and loon, ““Mvamled \n",
      "they\n",
      "===========================================================================================\n",
      "35\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 17000/256485, 6.628067918201844%\n",
      "tensor(1.9616, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE Lris m.. W. snel \n",
      "caces. The tlet inch seapedshe\n",
      "===========================================================================================\n",
      "36\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 17500/256485, 6.823011092266603%\n",
      "tensor(1.9681, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harre you hed jow. Ay Againg fled \n",
      "surd, Harry you \n",
      "===========================================================================================\n",
      "37\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 18000/256485, 7.017954266331364%\n",
      "tensor(2.0095, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry. Dare had ntond it of the Vermow the wass — B\n",
      "===========================================================================================\n",
      "38\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 18500/256485, 7.212897440396125%\n",
      "tensor(2.0117, device='cuda:0', grad_fn=<AddBackward0>)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========================================= TEXT ============================================\n",
      "Ha. \n",
      "doung congdo’s on they yee Potixgh ounn — \n",
      "Iak\n",
      "===========================================================================================\n",
      "39\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 19000/256485, 7.407840614460885%\n",
      "tensor(2.0206, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harrye goterirg revim ofly \n",
      "seining Herry domb, the\n",
      "===========================================================================================\n",
      "40\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 19500/256485, 7.602783788525645%\n",
      "tensor(1.9923, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE. \n",
      "““Not you of houts, she Dontuslnoming dreych \n",
      "\n",
      "===========================================================================================\n",
      "41\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 20000/256485, 7.7977269625904055%\n",
      "tensor(2.0060, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harrying had the kbom comlyoowtanengle nothing magl\n",
      "===========================================================================================\n",
      "42\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 20500/256485, 7.992670136655165%\n",
      "tensor(2.0141, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harres mewfile. \n",
      "“They’rd \n",
      "of statar onet dins byop\n",
      "===========================================================================================\n",
      "43\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 21000/256485, 8.187613310719925%\n",
      "tensor(2.0024, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry sho geee sigch, ghougking wald lash \n",
      "jutt \n",
      "op\n",
      "===========================================================================================\n",
      "44\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 21500/256485, 8.382556484784685%\n",
      "tensor(2.0084, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H. Wyin tro got park op Md farriedly, to to but \n",
      "\n",
      "p\n",
      "===========================================================================================\n",
      "45\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 22000/256485, 8.577499658849446%\n",
      "tensor(1.9838, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Ha. \n",
      "Yeus Hapry punet mall and therins warst shasye\n",
      "===========================================================================================\n",
      "46\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 22500/256485, 8.772442832914205%\n",
      "tensor(1.9775, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He tre baus the he whutceins. \n",
      "he they thei-shing a\n",
      "===========================================================================================\n",
      "47\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 23000/256485, 8.967386006978966%\n",
      "tensor(2.0015, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry and Ron. \n",
      "“Wablinc; — ” us theren’s a it. \n",
      "“I\n",
      "===========================================================================================\n",
      "48\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 23500/256485, 9.162329181043726%\n",
      "tensor(2.0099, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harrowe out foone. you Harry in opply call of fould\n",
      "===========================================================================================\n",
      "49\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 24000/256485, 9.357272355108485%\n",
      "tensor(2.0040, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He \n",
      "y ully was tevem. “The gat abratilged as vert f\n",
      "===========================================================================================\n",
      "50\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 24500/256485, 9.552215529173246%\n",
      "tensor(1.9792, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hagry and. Barne’s yoar \n",
      "bun to and dond — \n",
      "couldel\n",
      "===========================================================================================\n",
      "51\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 25000/256485, 9.747158703238005%\n",
      "tensor(1.9762, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE ryid Ron. Allfars, yey it?” shousesmon a loume’s\n",
      "===========================================================================================\n",
      "52\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 25500/256485, 9.942101877302765%\n",
      "tensor(1.9963, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Ho sand wat \n",
      "and the frabed at hadded him. H. Harry\n",
      "===========================================================================================\n",
      "53\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 26000/256485, 10.137045051367528%\n",
      "tensor(1.9852, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harryeen ghtile. “Ai Pe!” \n",
      "“I you’shrey and the ite\n",
      "===========================================================================================\n",
      "54\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 26500/256485, 10.331988225432287%\n",
      "tensor(1.9817, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H. I to had be \n",
      "dowche it ily was poodling, \n",
      "qurrt.\n",
      "===========================================================================================\n",
      "55\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 27000/256485, 10.526931399497046%\n",
      "tensor(1.9615, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HYrode syougle. I’velly dent aspeapped shome letred\n",
      "===========================================================================================\n",
      "56\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 27500/256485, 10.721874573561808%\n",
      "tensor(1.9570, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry \n",
      "gurasning dew pmabuthom and ant’s whed thwat\n",
      "===========================================================================================\n",
      "57\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 28000/256485, 10.916817747626567%\n",
      "tensor(1.9391, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hary — slasarm tlapgher with a drand lowers had was\n",
      "===========================================================================================\n",
      "58\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 28500/256485, 11.111760921691328%\n",
      "tensor(1.9276, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE. Do bin’t he of an laor Mackine sor, the and. Th\n",
      "===========================================================================================\n",
      "59\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 29000/256485, 11.306704095756087%\n",
      "tensor(1.9372, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hared ingle woleed. “Bray. \n",
      "m’per It canghaed sear.\n",
      "===========================================================================================\n",
      "60\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 29500/256485, 11.501647269820847%\n",
      "tensor(1.9275, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry we \n",
      "an tam fert od though? Weacked. Handine r\n",
      "===========================================================================================\n",
      "61\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 30000/256485, 11.696590443885608%\n",
      "tensor(1.9296, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harrie stohine ir alry fort.” \n",
      "He decn, then’s one \n",
      "===========================================================================================\n",
      "62\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 30500/256485, 11.891533617950367%\n",
      "tensor(1.9193, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He llede came some. Whis that mise. “I’f you oaredr\n",
      "===========================================================================================\n",
      "63\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 31000/256485, 12.086476792015127%\n",
      "tensor(1.9196, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H..” \n",
      "“Droy her mogloy the \n",
      "quing a cantinh Slather\n",
      "===========================================================================================\n",
      "64\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 31500/256485, 12.281419966079888%\n",
      "tensor(1.9103, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harrly a.” \n",
      "“Oovered. “Yet intiding ’h hocksow spac\n",
      "===========================================================================================\n",
      "65\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 32000/256485, 12.476363140144647%\n",
      "tensor(1.9171, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He wny \n",
      "hart hamer the him we reacabeme, a wnives w\n",
      "===========================================================================================\n",
      "66\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 32500/256485, 12.671306314209406%\n",
      "tensor(1.9062, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry in with sain to” to theme the becting doing \n",
      "\n",
      "===========================================================================================\n",
      "67\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 33000/256485, 12.866249488274168%\n",
      "tensor(1.8993, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hardy to over, sitting lioht.” \n",
      "Harry dick the migr\n",
      "===========================================================================================\n",
      "68\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 33500/256485, 13.061192662338927%\n",
      "tensor(1.9081, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry you malllis bected the feme would to he Hagri\n",
      "===========================================================================================\n",
      "69\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 34000/256485, 13.256135836403688%\n",
      "tensor(1.8824, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry be seme \n",
      "yet he hand some ald theled. \n",
      "“Oun, \n",
      "===========================================================================================\n",
      "70\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 34500/256485, 13.451079010468447%\n",
      "tensor(1.8754, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harryougow twromoor?” \n",
      "Who tome Slass!!” \n",
      "Ars exaly\n",
      "===========================================================================================\n",
      "71\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 35000/256485, 13.646022184533207%\n",
      "tensor(1.8718, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He ho chound trongy with to a argon’t w furs not ..\n",
      "===========================================================================================\n",
      "72\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 35500/256485, 13.840965358597968%\n",
      "tensor(1.8558, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harowe e’ve aud wile weowe do. He mores excanted.” \n",
      "===========================================================================================\n",
      "73\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 36000/256485, 14.035908532662727%\n",
      "tensor(1.8531, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry un dny?” \n",
      "“Od said. Locks. “I reess stalle. R\n",
      "===========================================================================================\n",
      "74\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 36500/256485, 14.230851706727488%\n",
      "tensor(1.8767, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hat you ell \n",
      "hext a dackly a prow?” \n",
      "Hurring with a\n",
      "===========================================================================================\n",
      "75\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 37000/256485, 14.42579488079225%\n",
      "tensor(1.8740, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Ht. \n",
      "on in anow to ask’s on tilu,” eveed look drepp\n",
      "===========================================================================================\n",
      "76\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===========================================================================================\n",
      "Update Step: 37500/256485, 14.62073805485701%\n",
      "tensor(1.8828, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He \n",
      "ret corf theided it vereme \n",
      "ald allye, they for\n",
      "===========================================================================================\n",
      "77\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 38000/256485, 14.81568122892177%\n",
      "tensor(1.8875, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry FAnden, and Pippeen moved they off \n",
      "Dumbly. T\n",
      "===========================================================================================\n",
      "78\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 38500/256485, 15.010624402986531%\n",
      "tensor(1.9322, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry bambouce in forenage the wind the Drizing lis\n",
      "===========================================================================================\n",
      "79\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 39000/256485, 15.20556757705129%\n",
      "tensor(1.9427, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harryesturd Unnh to a \n",
      "at that e suve honly the rea\n",
      "===========================================================================================\n",
      "80\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 39500/256485, 15.40051075111605%\n",
      "tensor(1.9356, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE Prot. I mago inge sot of hold har glapay. sarm.”\n",
      "===========================================================================================\n",
      "81\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 40000/256485, 15.595453925180811%\n",
      "tensor(1.9347, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hary but \n",
      "tame erne — it a \n",
      "to crade \n",
      "he had deam h\n",
      "===========================================================================================\n",
      "82\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 40500/256485, 15.79039709924557%\n",
      "tensor(1.9554, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hary. Bustimefwing avist of eythirgurdnigh aus Cunt\n",
      "===========================================================================================\n",
      "83\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 41000/256485, 15.98534027331033%\n",
      "tensor(1.9716, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harrowed throws conent ray. I he’d. \n",
      "An to ye pulad\n",
      "===========================================================================================\n",
      "84\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 41500/256485, 16.18028344737509%\n",
      "tensor(1.9427, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Her. Harry. \n",
      "Harry eared. ...” \n",
      "“Aed, look.. \n",
      "“ the\n",
      "===========================================================================================\n",
      "85\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 42000/256485, 16.37522662143985%\n",
      "tensor(1.9168, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry telliserl’s seeved to them at plize for \n",
      "mook\n",
      "===========================================================================================\n",
      "86\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 42500/256485, 16.57016979550461%\n",
      "tensor(1.8877, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE... \n",
      "“He \n",
      "she was Hor, areace, had \n",
      "groor a was t\n",
      "===========================================================================================\n",
      "87\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 43000/256485, 16.76511296956937%\n",
      "tensor(1.8567, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hurry it the Wouddhor’s stinighso whappel the \n",
      "reap\n",
      "===========================================================================================\n",
      "88\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 43500/256485, 16.96005614363413%\n",
      "tensor(1.8513, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He \n",
      "rogin \n",
      "Anclled the half they you brow latestter\n",
      "===========================================================================================\n",
      "89\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 44000/256485, 17.15499931769889%\n",
      "tensor(1.8863, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He. \n",
      "rou, steant it. Profe goiny he wher tafos you \n",
      "===========================================================================================\n",
      "90\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 44500/256485, 17.349942491763652%\n",
      "tensor(1.8758, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He \n",
      "Harfy himdars cheak! Ore of, and look to in her\n",
      "===========================================================================================\n",
      "91\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 45000/256485, 17.54488566582841%\n",
      "tensor(1.8672, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Haris con’s histling int way yed his non thinged th\n",
      "===========================================================================================\n",
      "92\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 45500/256485, 17.73982883989317%\n",
      "tensor(1.8653, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He \n",
      "Herpives.” sain wint but miverring the babbed \n",
      "\n",
      "===========================================================================================\n",
      "93\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 46000/256485, 17.934772013957932%\n",
      "tensor(1.8816, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hary mrowed a \n",
      "stavion \n",
      "us had \n",
      "up to and \n",
      "doodlats\n",
      "===========================================================================================\n",
      "94\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 46500/256485, 18.12971518802269%\n",
      "tensor(1.8957, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He \n",
      "yer Lothroums furver Blarkly did of thered wors\n",
      "===========================================================================================\n",
      "95\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 47000/256485, 18.32465836208745%\n",
      "tensor(1.8601, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H \n",
      "rook Lockud \n",
      "becking I knom \n",
      "and busunh,” said L\n",
      "===========================================================================================\n",
      "96\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 47500/256485, 18.519601536152212%\n",
      "tensor(1.8533, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hary in walk of the killes. \n",
      "Aunt neveronees. Int —\n",
      "===========================================================================================\n",
      "97\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 48000/256485, 18.71454471021697%\n",
      "tensor(1.8592, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry. It wasteomingoushedmoon becr thoight beas go\n",
      "===========================================================================================\n",
      "98\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 48500/256485, 18.90948788428173%\n",
      "tensor(1.8420, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H.” \n",
      "Hermione. \n",
      "fild \n",
      "ell the ffope in Ron, that he\n",
      "===========================================================================================\n",
      "99\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 49000/256485, 19.104431058346492%\n",
      "tensor(1.8435, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harro’s hyourh the ont at yourtt. Harry nond Hermig\n",
      "===========================================================================================\n",
      "100\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 49500/256485, 19.29937423241125%\n",
      "tensor(1.8784, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hary \n",
      "ne the mad, nostids a staring. \n",
      "“Doorgateren \n",
      "===========================================================================================\n",
      "101\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 50000/256485, 19.49431740647601%\n",
      "tensor(1.9060, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hear fin’t we kattedrer for withing?” \n",
      "“That-ill. \n",
      "\n",
      "===========================================================================================\n",
      "102\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 50500/256485, 19.689260580540772%\n",
      "tensor(1.8869, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H’. dor was he. ho had ceffitcofe) up he rous, was \n",
      "===========================================================================================\n",
      "103\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 51000/256485, 19.88420375460553%\n",
      "tensor(1.8843, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE tien’s his trobouted \n",
      "his smandored \n",
      "the one sli\n",
      "===========================================================================================\n",
      "104\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 51500/256485, 20.079146928670294%\n",
      "tensor(1.8648, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harud \n",
      "rosbused spent of hest to mamming said Fudor\n",
      "===========================================================================================\n",
      "105\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 52000/256485, 20.274090102735055%\n",
      "tensor(1.8566, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He “re wis dyowing cor ofs \n",
      "firalt — \n",
      "“Deckboct sha\n",
      "===========================================================================================\n",
      "106\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 52500/256485, 20.469033276799813%\n",
      "tensor(1.8542, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry a wrout horme the \n",
      "gonedy Het.” \n",
      "“Moneing. Th\n",
      "===========================================================================================\n",
      "107\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 53000/256485, 20.663976450864574%\n",
      "tensor(1.8643, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H’. Ito hil. \n",
      "“Harry in thing to Praver,” haartent \n",
      "===========================================================================================\n",
      "108\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 53500/256485, 20.858919624929335%\n",
      "tensor(1.8644, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harmbess butch bisssorg ook \n",
      "mage. he \n",
      "his have, un\n",
      "===========================================================================================\n",
      "109\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 54000/256485, 21.053862798994093%\n",
      "tensor(1.8828, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har. The. \n",
      "“Kho to Ron come and \n",
      "I cars. \n",
      "Harry Hon\n",
      "===========================================================================================\n",
      "110\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 54500/256485, 21.248805973058854%\n",
      "tensor(1.8606, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HY’n pagitued she the . . \n",
      "round, haarled the Hagri\n",
      "===========================================================================================\n",
      "111\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 55000/256485, 21.443749147123615%\n",
      "tensor(1.8439, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE. rous their. HHarry, sight fintid. \n",
      "Her. Doneys \n",
      "===========================================================================================\n",
      "112\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 55500/256485, 21.638692321188373%\n",
      "tensor(1.8552, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har \n",
      "Her ite,” said Spaired ger. \n",
      "“Arn, I’d clamS o\n",
      "===========================================================================================\n",
      "113\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 56000/256485, 21.833635495253134%\n",
      "tensor(1.8838, device='cuda:0', grad_fn=<AddBackward0>)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========================================= TEXT ============================================\n",
      "HA. Iy Qurringove, padististing had hermed intind h\n",
      "===========================================================================================\n",
      "114\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 56500/256485, 22.028578669317895%\n",
      "tensor(1.8988, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE. \n",
      "“Pryfeathiambood At ext to foully of histaincr\n",
      "===========================================================================================\n",
      "115\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 57000/256485, 22.223521843382656%\n",
      "tensor(1.8676, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H rras for belfint was seabpleted. AN!” he \n",
      "and sai\n",
      "===========================================================================================\n",
      "116\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 57500/256485, 22.418465017447414%\n",
      "tensor(1.8332, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He cat. ... \n",
      "Harry was shing \n",
      "the man pay.” \n",
      "Ighty \n",
      "===========================================================================================\n",
      "117\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 58000/256485, 22.613408191512175%\n",
      "tensor(1.8080, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE.... \n",
      "But — ” seashes mif his stoom. Harry looke \n",
      "===========================================================================================\n",
      "118\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 58500/256485, 22.808351365576936%\n",
      "tensor(1.8066, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Haro in the the some.. \n",
      "“Lupinis a doing with himmo\n",
      "===========================================================================================\n",
      "119\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 59000/256485, 23.003294539641693%\n",
      "tensor(1.8289, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He irenshishes wanged araw acroye scaille \n",
      "in to to\n",
      "===========================================================================================\n",
      "120\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 59500/256485, 23.198237713706455%\n",
      "tensor(1.8201, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H Blow Lugreftinged. Sheack., \n",
      "“Eronger urilge stol\n",
      "===========================================================================================\n",
      "121\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 60000/256485, 23.393180887771216%\n",
      "tensor(1.8243, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE. \n",
      "“Bried beters wine, . . “yo the muin byther hi\n",
      "===========================================================================================\n",
      "122\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 60500/256485, 23.588124061835973%\n",
      "tensor(1.8153, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Haro fure undere \n",
      "be ius break sare Potrrides. he s\n",
      "===========================================================================================\n",
      "123\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 61000/256485, 23.783067235900734%\n",
      "tensor(1.8217, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He \n",
      "the mast you said this beachining \n",
      "was the ram.\n",
      "===========================================================================================\n",
      "124\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 61500/256485, 23.978010409965496%\n",
      "tensor(1.7968, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hary las tubbound whakedseats but booressaus his pu\n",
      "===========================================================================================\n",
      "125\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 62000/256485, 24.172953584030253%\n",
      "tensor(1.7577, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He Har igh speaposcherser the tome bithed seer atri\n",
      "===========================================================================================\n",
      "126\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 62500/256485, 24.367896758095014%\n",
      "tensor(1.7666, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har on. \n",
      "Bowp in there try, I you!” \n",
      "“Who gliithle.\n",
      "===========================================================================================\n",
      "127\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 63000/256485, 24.562839932159775%\n",
      "tensor(1.7819, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hard Haid. He he be war,” Dumble of lain. Nowes the\n",
      "===========================================================================================\n",
      "128\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 63500/256485, 24.757783106224533%\n",
      "tensor(1.8307, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hard what diddeds,” \n",
      "he some Dinchine. “The goter s\n",
      "===========================================================================================\n",
      "129\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 64000/256485, 24.952726280289294%\n",
      "tensor(1.8670, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE. Ill he mad on at pettan tim,” so sirining him a\n",
      "===========================================================================================\n",
      "130\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 64500/256485, 25.147669454354055%\n",
      "tensor(1.8481, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har sarqued, the from, thrack \n",
      "midning. \n",
      "Ther-gor a\n",
      "===========================================================================================\n",
      "131\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 65000/256485, 25.342612628418813%\n",
      "tensor(1.8524, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He rras \n",
      "dTin’t? But ont of could that \n",
      "mometiius. \n",
      "===========================================================================================\n",
      "132\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 65500/256485, 25.537555802483574%\n",
      "tensor(1.8648, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He  ur for \n",
      "Acools. And Hagry \n",
      "whey cullc pecle eve\n",
      "===========================================================================================\n",
      "133\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 66000/256485, 25.732498976548335%\n",
      "tensor(1.8629, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har. Hare tomurize darder Quacser saiding tostedere\n",
      "===========================================================================================\n",
      "134\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 66500/256485, 25.927442150613096%\n",
      "tensor(1.8407, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har dy. Wyottire Dudin Vors bucks. Wevery amage emc\n",
      "===========================================================================================\n",
      "135\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 67000/256485, 26.122385324677854%\n",
      "tensor(1.8323, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE Woathars. \n",
      "“Whak’s all the pave so witguid, it s\n",
      "===========================================================================================\n",
      "136\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 67500/256485, 26.317328498742615%\n",
      "tensor(1.8448, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE. ritew to habld with the dillsent the ghtoly?” s\n",
      "===========================================================================================\n",
      "137\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 68000/256485, 26.512271672807376%\n",
      "tensor(1.8582, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harm te \n",
      "lick the wample outs. \n",
      "Dernund,” Mingo; he\n",
      "===========================================================================================\n",
      "138\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 68500/256485, 26.707214846872134%\n",
      "tensor(1.8591, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har Harry, \n",
      "the Bepled spernone and af Fundoy nat a\n",
      "===========================================================================================\n",
      "139\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 69000/256485, 26.902158020936895%\n",
      "tensor(1.8818, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE He his \n",
      "shaply?” A Mr.” \n",
      "“You was areeal wand \n",
      "a\n",
      "===========================================================================================\n",
      "140\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 69500/256485, 27.097101195001656%\n",
      "tensor(1.8929, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He wail for under, the seee. Pesch, forioued shinke\n",
      "===========================================================================================\n",
      "141\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 70000/256485, 27.292044369066414%\n",
      "tensor(1.9235, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har. Harry’s wads the toulled ougsolf though.” \n",
      "Qui\n",
      "===========================================================================================\n",
      "142\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 70500/256485, 27.486987543131175%\n",
      "tensor(1.9456, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE Dut \n",
      "booking lovk- chardilt fleat, somen, whowin\n",
      "===========================================================================================\n",
      "143\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 71000/256485, 27.681930717195936%\n",
      "tensor(1.8947, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE \n",
      "reen tHa ging Bourly, as gow there rould Mand a\n",
      "===========================================================================================\n",
      "144\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 71500/256485, 27.876873891260693%\n",
      "tensor(1.8503, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE.. This ian a his in a dide to Manced to \n",
      "thrale \n",
      "===========================================================================================\n",
      "145\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 72000/256485, 28.071817065325455%\n",
      "tensor(1.8429, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE I mesarkaw has the Dakg \n",
      "their tessed wizalds, w\n",
      "===========================================================================================\n",
      "146\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 72500/256485, 28.266760239390216%\n",
      "tensor(1.8265, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hary’t lithers lookged the; to belied deed, in on t\n",
      "===========================================================================================\n",
      "147\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 73000/256485, 28.461703413454977%\n",
      "tensor(1.8318, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE.. O the the rasing \n",
      "dider ard Mr. Distal sudch \n",
      "\n",
      "===========================================================================================\n",
      "148\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 73500/256485, 28.65664658751974%\n",
      "tensor(1.8350, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harriet \n",
      "Keared to bo), the I, Sidicgans lainst to \n",
      "===========================================================================================\n",
      "149\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 74000/256485, 28.8515897615845%\n",
      "tensor(1.8328, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry’s suded Weatler a go’s like a tidn’t befostag\n",
      "===========================================================================================\n",
      "150\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 74500/256485, 29.04653293564926%\n",
      "tensor(1.8576, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE Ithor Mrsboogand has? \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "“Ohe seaff bon, frobble!”\n",
      "===========================================================================================\n",
      "151\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 75000/256485, 29.24147610971402%\n",
      "tensor(1.8676, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har e. rrapbew ... \n",
      "“Proness.” \n",
      "the pixsow thourd D\n",
      "===========================================================================================\n",
      "152\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 75500/256485, 29.436419283778783%\n",
      "tensor(1.8804, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hard histled Plele of the \n",
      "hisping, \n",
      "wa mort warpin\n",
      "===========================================================================================\n",
      "153\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 76000/256485, 29.63136245784354%\n",
      "tensor(1.8759, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harm!” \n",
      "youre shey, perping seeed wis willed Magily\n",
      "===========================================================================================\n",
      "154\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 76500/256485, 29.8263056319083%\n",
      "tensor(1.8483, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE Rindhrough to could. \n",
      "“Aillely Mabby pout \n",
      "oned \n",
      "===========================================================================================\n",
      "155\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 77000/256485, 30.021248805973062%\n",
      "tensor(1.8368, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry. ... fid thend they \n",
      "edsull \n",
      "saffy narnile to\n",
      "===========================================================================================\n",
      "156\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 77500/256485, 30.21619198003782%\n",
      "tensor(1.8296, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HLRIT, \n",
      "9heme \n",
      "backed reald of freemuted beaping a \n",
      "===========================================================================================\n",
      "157\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 78000/256485, 30.41113515410258%\n",
      "tensor(1.8499, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He Gug, It that dudfnow, the brem goict oncants. \n",
      "“\n",
      "===========================================================================================\n",
      "158\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 78500/256485, 30.606078328167342%\n",
      "tensor(1.8586, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He mowndaware it, Harry a me telind. “I found ye \n",
      "t\n",
      "===========================================================================================\n",
      "159\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 79000/256485, 30.8010215022321%\n",
      "tensor(1.8472, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harmy she surry by Aw happertiet, to dowash of he o\n",
      "===========================================================================================\n",
      "160\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 79500/256485, 30.99596467629686%\n",
      "tensor(1.8334, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H Dis of . D raffy meptiin.” Harry’s ceateafly \n",
      "ren\n",
      "===========================================================================================\n",
      "161\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 80000/256485, 31.190907850361622%\n",
      "tensor(1.8287, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He. \n",
      "said a set?” \n",
      "Suppec all to \n",
      "call was caking t\n",
      "===========================================================================================\n",
      "162\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 80500/256485, 31.38585102442638%\n",
      "tensor(1.8271, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry surden qumper whatoos frireing the locsed. \n",
      "“\n",
      "===========================================================================================\n",
      "163\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 81000/256485, 31.58079419849114%\n",
      "tensor(1.8326, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry beet I moken, \n",
      "“us. The she didnost. “Bloth \n",
      "\n",
      "===========================================================================================\n",
      "164\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 81500/256485, 31.775737372555902%\n",
      "tensor(1.8025, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Haro? . . \n",
      "bread the enteor are than’y at Tor!” \n",
      "Th\n",
      "===========================================================================================\n",
      "165\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 82000/256485, 31.97068054662066%\n",
      "tensor(1.8197, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry sered by shull feer leacss buriding — “Roult \n",
      "===========================================================================================\n",
      "166\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 82500/256485, 32.165623720685424%\n",
      "tensor(1.8321, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harm s her the . \n",
      "derainedoro \n",
      "Harmy fere Verrinned\n",
      "===========================================================================================\n",
      "167\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 83000/256485, 32.36056689475018%\n",
      "tensor(1.8327, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har ha \n",
      "drowing he drabanom \n",
      "zow you,” Harry Harry \n",
      "===========================================================================================\n",
      "168\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 83500/256485, 32.55551006881494%\n",
      "tensor(1.8128, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry ire case cone the row — whis dising \n",
      "epasely \n",
      "===========================================================================================\n",
      "169\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 84000/256485, 32.7504532428797%\n",
      "tensor(1.8186, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "His light — \n",
      ".. So izarry to the had \n",
      "and holfer a \n",
      "===========================================================================================\n",
      "170\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 84500/256485, 32.94539641694446%\n",
      "tensor(1.8115, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harrgw-. \n",
      "Hermiesy a a traling Pitting sippien. \n",
      "“H\n",
      "===========================================================================================\n",
      "171\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 85000/256485, 33.14033959100922%\n",
      "tensor(1.8027, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE.E \n",
      "Harry.” \n",
      "“Whes had \n",
      "hadd \n",
      "have all at himstel\n",
      "===========================================================================================\n",
      "172\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 85500/256485, 33.335282765073984%\n",
      "tensor(1.8021, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE Ron, an’ve so muroun hhisten. “Vourn!” be outt l\n",
      "===========================================================================================\n",
      "173\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 86000/256485, 33.53022593913874%\n",
      "tensor(1.7873, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He rron Porken of Ron, Con was up but le ghouge, co\n",
      "===========================================================================================\n",
      "174\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 86500/256485, 33.7251691132035%\n",
      "tensor(1.8072, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry Harry will ne, noveng stobling as pully. Nevi\n",
      "===========================================================================================\n",
      "175\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 87000/256485, 33.92011228726826%\n",
      "tensor(1.8020, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harly — \n",
      "youde frelim!” said Ron sime fagening raya\n",
      "===========================================================================================\n",
      "176\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 87500/256485, 34.11505546133302%\n",
      "tensor(1.7942, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He tir fow hadd puthous, they rogs again, than coul\n",
      "===========================================================================================\n",
      "177\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 88000/256485, 34.30999863539778%\n",
      "tensor(1.8060, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hor ward Harry hat the \n",
      "mowind tratter, mos ghor he\n",
      "===========================================================================================\n",
      "178\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 88500/256485, 34.504941809462544%\n",
      "tensor(1.7820, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry seerfing shar going opered doightt bumtruive.\n",
      "===========================================================================================\n",
      "179\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 89000/256485, 34.699884983527305%\n",
      "tensor(1.8036, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HO.. \n",
      "Har Harruppeening a aliw afowing tain relust \n",
      "===========================================================================================\n",
      "180\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 89500/256485, 34.89482815759206%\n",
      "tensor(1.8240, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry Flent, bafy atillbone list of Harry, you and \n",
      "===========================================================================================\n",
      "181\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 90000/256485, 35.08977133165682%\n",
      "tensor(1.8171, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Haris out ore \n",
      "t’ap thed youds’ stree \n",
      "smother, nob\n",
      "===========================================================================================\n",
      "182\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 90500/256485, 35.28471450572158%\n",
      "tensor(1.8068, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE boh fer over \n",
      "he was beh had uncer, this that a \n",
      "===========================================================================================\n",
      "183\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 91000/256485, 35.47965767978634%\n",
      "tensor(1.8209, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He. I the cereretting of take noser-ght now corce t\n",
      "===========================================================================================\n",
      "184\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 91500/256485, 35.6746008538511%\n",
      "tensor(1.8101, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har. Hall to ange, not lall is \n",
      "donn bangelded!” \n",
      "“\n",
      "===========================================================================================\n",
      "185\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 92000/256485, 35.869544027915865%\n",
      "tensor(1.8201, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He, least in that that off the oll’s the was \n",
      "unvay\n",
      "===========================================================================================\n",
      "186\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 92500/256485, 36.06448720198062%\n",
      "tensor(1.8198, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harrun abvendn’t flapped of this \n",
      "smatterfwed and t\n",
      "===========================================================================================\n",
      "187\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 93000/256485, 36.25943037604538%\n",
      "tensor(1.8049, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry, . Hyer have as sime, jooword lisk frot allig\n",
      "===========================================================================================\n",
      "188\n",
      "\n",
      "\n",
      "===========================================================================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Update Step: 93500/256485, 36.45437355011014%\n",
      "tensor(1.7839, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har ’N. \n",
      "He and stupning — a Greant, ham to sems th\n",
      "===========================================================================================\n",
      "189\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 94000/256485, 36.6493167241749%\n",
      "tensor(1.7928, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry sarye!” Harry car they he snurnen \n",
      "his your,”\n",
      "===========================================================================================\n",
      "190\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 94500/256485, 36.84425989823966%\n",
      "tensor(1.7981, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hary whotersand tood and watkeling charcroye up on \n",
      "===========================================================================================\n",
      "191\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 95000/256485, 37.039203072304424%\n",
      "tensor(1.7794, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He buthing \n",
      "and the At wearmown.” \n",
      "Harry out was — \n",
      "===========================================================================================\n",
      "192\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 95500/256485, 37.23414624636918%\n",
      "tensor(1.8037, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry Sarry head, brimit-not you litter gow aloul P\n",
      "===========================================================================================\n",
      "193\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 96000/256485, 37.42908942043394%\n",
      "tensor(1.7960, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har warmever, and to Harry Harry hore pro was yel, \n",
      "===========================================================================================\n",
      "194\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 96500/256485, 37.6240325944987%\n",
      "tensor(1.7891, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harud qurush show. “They,” \n",
      "Shrumbed, the moghtlece\n",
      "===========================================================================================\n",
      "195\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 97000/256485, 37.81897576856346%\n",
      "tensor(1.7801, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He Dur. Wiry boncery the Grogger. “Sothy armunly-Pr\n",
      "===========================================================================================\n",
      "196\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 97500/256485, 38.01391894262822%\n",
      "tensor(1.7931, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har \n",
      "Crrets,” say lander wifffuech hemssing \n",
      "and ha\n",
      "===========================================================================================\n",
      "197\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 98000/256485, 38.208862116692984%\n",
      "tensor(1.8002, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE Ron fieds. \n",
      "“IN the Koncelvedn’t the lithered. “\n",
      "===========================================================================================\n",
      "198\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 98500/256485, 38.403805290757745%\n",
      "tensor(1.7508, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hare an’ Harry. Whe mally howle richer andwerm \n",
      "tol\n",
      "===========================================================================================\n",
      "199\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 99000/256485, 38.5987484648225%\n",
      "tensor(1.7213, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE Rou’ hald Obessore \n",
      "dose exupbearpoon groued the\n",
      "===========================================================================================\n",
      "200\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 99500/256485, 38.79369163888726%\n",
      "tensor(1.7382, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Harry Klom, blown up in him heas nartly was voup to\n",
      "===========================================================================================\n",
      "201\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 100000/256485, 38.98863481295202%\n",
      "tensor(1.7339, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Hary \n",
      "Thand the his?” said Ned mmintelled and she f\n",
      "===========================================================================================\n",
      "202\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 100500/256485, 39.18357798701678%\n",
      "tensor(1.7641, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har wat mertend vincicas to Harry ore. \n",
      "“Ron his \n",
      "m\n",
      "===========================================================================================\n",
      "203\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 101000/256485, 39.378521161081544%\n",
      "tensor(1.7626, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har. ” \n",
      "that ther De \n",
      "are cland hamrord.” \n",
      "“Well — \n",
      "===========================================================================================\n",
      "204\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 101500/256485, 39.573464335146305%\n",
      "tensor(1.7509, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "HE. . . . . He seer and be wenten. \n",
      "“Fireter at the\n",
      "===========================================================================================\n",
      "205\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 102000/256485, 39.76840750921106%\n",
      "tensor(1.7743, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "He . Cor teaxpett!” \n",
      "The grirely ratch Dalble, and \n",
      "===========================================================================================\n",
      "206\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 102500/256485, 39.96335068327583%\n",
      "tensor(1.7851, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H.. ret and inly pridie don’t he cooking, his groug\n",
      "===========================================================================================\n",
      "207\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 103000/256485, 40.15829385734059%\n",
      "tensor(1.7802, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "H . to he He’rff befur. \n",
      "“Yeat of Roght,” Hermid pe\n",
      "===========================================================================================\n",
      "208\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 103500/256485, 40.35323703140535%\n",
      "tensor(1.7600, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har ’d bes and Brob. Whan leatzice. \n",
      "Haldie some-ow\n",
      "===========================================================================================\n",
      "209\n",
      "\n",
      "\n",
      "===========================================================================================\n",
      "Update Step: 104000/256485, 40.54818020547011%\n",
      "tensor(1.7944, device='cuda:0', grad_fn=<AddBackward0>)\n",
      "========================================= TEXT ============================================\n",
      "Har haireseod, whese at resk your the ronders, Ron \n",
      "===========================================================================================\n"
     ]
    }
   ],
   "source": [
    "train(model, 2, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generated_seq = synthesize_text(model, 'H', 1000)\n",
    "print(generated_seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prime the network with a context before generation,\n",
    "# i.e only pass the hidden state forward, not the predicted\n",
    "# character.\n",
    "def synthesize_text2(model, prime, length):\n",
    "    model.eval()\n",
    "    generated_seq = [char for char in prime]\n",
    "    hidden = model.init_hidden(1)\n",
    "    \n",
    "    #Used to prime the network\n",
    "    for char in prime:\n",
    "        x0 = np.zeros(K, dtype=np.float32)\n",
    "        x0[c2i[char]] = 1\n",
    "        x0 = torch.from_numpy(x0)\n",
    "        x0 = x0.view(1,1,K).to(device)\n",
    "        c, hidden = model(x0, hidden)\n",
    "    \n",
    "    c, i, x0 = get_next_char(c)\n",
    "    generated_seq.append(i)\n",
    "    \n",
    "    #x0 = np.zeros(K, dtype=np.float32)\n",
    "    x0 = x0.view(1,1,K).to(device)\n",
    "    \n",
    "    for i in range(length):\n",
    "        #print(\"x0 shape: \" + str(x0.shape))\n",
    "        #print(\"hidden shape: \" + str(hidden.shape))\n",
    "        s, hidden = model(x0, hidden)\n",
    "        i, c, x0 = get_next_char(s)\n",
    "        generated_seq.append(c)\n",
    "    model.train()\n",
    "    \n",
    "   # returnString = \"\"\n",
    "    \n",
    "   # for c in generated_seq:\n",
    "       # returnString += str(c)\n",
    "    \n",
    "    return ''.join(generated_seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generated_seq = synthesize_text2(model, 'Harry Potter', 1000)\n",
    "print(generated_seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "xaxis = range(len(lossarr))\n",
    "print(lossarr)\n",
    "plt.plot(xaxis,lossarr)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for str in strarr:\n",
    "    print(str)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change the name, for saving multiple files\n",
    "model_name = 'vanilla_rnn_all_books.net'\n",
    "\n",
    "checkpoint = {'hidden_size': model.hidden_size,\n",
    "              'n_layers': model.n_layers,\n",
    "              'state_dict': model.state_dict(),\n",
    "              'tokens': chars}\n",
    "\n",
    "with open(model_name, 'wb') as f:\n",
    "    torch.save(checkpoint, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom Accuracy Measure, measures amount of actual (correct) words generated.\n",
    "import re\n",
    "#generated_words = generated_seq.split()\n",
    "trials = 100\n",
    "mean_actual_word_count = 0\n",
    "f = open('goblet_book.txt').read()\n",
    "\n",
    "for i in range(100):\n",
    "    generated_words = re.findall(r\"[\\w']+\", synthesize_text2(model, 'Harry Potter', 1000))\n",
    "    #print(generated_words)\n",
    "    actual_words_count = 0\n",
    "    for word in generated_words: \n",
    "        if word in f:\n",
    "            #print(\"hey! this word matched!\")\n",
    "            #print(word)\n",
    "            actual_words_count += 1\n",
    "\n",
    "    actual_words_ratio = actual_words_count/len(generated_words)\n",
    "    mean_actual_word_count += actual_words_ratio\n",
    "    #print(\"synthesized \" + str(actual_words_count) + \" real words, out of \" + str(len(generated_words)) + \" attempts.\")\n",
    "    #print(\"actual word ratio: \" + str(actual_words_ratio))\n",
    "\n",
    "print(\"Mean of correct words generated: \")\n",
    "print(mean_actual_word_count/trials)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}