stateful-LSTM-LM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import re\n",
    "import pickle\n",
    "import numpy as np\n",
    "from os import path\n",
    "from math import exp\n",
    "from keras import optimizers\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.models import load_model\n",
    "from keras.layers import Dense\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Embedding\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import Input\n",
    "from keras.layers import TimeDistributed\n",
    "from keras.callbacks import History\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.callbacks import Callback\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "from keras.callbacks import ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## text preprocessing for cross sentence training\n",
    "def cross_sentence_preproc(text):\n",
    "    return text.replace('\\n',' <eos> ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## tokenize textfile and save as pickle file\n",
    "def tokenize_txt_cross_sentence(textfile):\n",
    "    ## open the file as read only\n",
    "    file = open(textfile, 'r', encoding='UTF-8')\n",
    "    ## read all text\n",
    "    text = file.read()\n",
    "    file.close()\n",
    "    print('\\nSample of original txt:\\n\\n', text[:300])\n",
    "    \n",
    "    ## run text preprocessing\n",
    "    text_proc = cross_sentence_preproc(text)\n",
    "    print('\\nSample of processed txt:\\n\\n', text_proc[:300])\n",
    "    print('\\nTotal tokens in text: %d' % len(text_proc.split()))\n",
    "    print('Unique tokens in text: %d' % len(set(text_proc.split())))\n",
    "    \n",
    "    ## fit tokenizer\n",
    "    tokenizer = Tokenizer(filters='', lower=False)\n",
    "    tokenizer.fit_on_texts([text_proc])\n",
    "    ## saving tokenizer\n",
    "    with open('tokenizer/cs-{0}.pickle'.format(path.splitext(path.basename(textfile))[0]), 'wb') as handle:\n",
    "        pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Converting text to lm training tensor for cross sentence LSTM\n",
    "def txt_to_tensor_cross_sent_LSTM(textfile, tokenizer, batch_size=20, seq_length=35):\n",
    "    \n",
    "    ## open the file as read only\n",
    "    file = open(textfile, 'r', encoding='UTF-8')\n",
    "    \n",
    "    ## read all text\n",
    "    text = file.read()\n",
    "    file.close()\n",
    "    print('\\nSample of original txt:\\n\\n', text[:300])\n",
    "    \n",
    "    ## run text preprocessing\n",
    "    text_proc = cross_sentence_preproc(text)\n",
    "    print('\\nSample of processed txt:\\n\\n', text_proc[:300])\n",
    "    print('\\nTotal tokens in text: %d' % len(text_proc.split()))\n",
    "    print('Unique tokens in text: %d' % len(set(text_proc.split())))\n",
    "    \n",
    "    ## load tokenizer\n",
    "    with open(tokenizer, 'rb') as handle:\n",
    "        tokenizer = pickle.load(handle)\n",
    "\n",
    "    ## vocabulary size\n",
    "    vocab_size = len(tokenizer.word_index) + 1\n",
    "    print('Vocab size: %d' % vocab_size)\n",
    "\n",
    "    ## coding text\n",
    "    text_coded = tokenizer.texts_to_sequences([text_proc])\n",
    "    text_coded_len = len(text_coded[0])\n",
    "    print('Coded text length:', text_coded_len)\n",
    "    \n",
    "    ## pad according to current batch size and seq length (stateful training requirement!)\n",
    "    padding_length = batch_size * seq_length * ( (text_coded_len // (batch_size * seq_length)) + 1 )\n",
    "    input_array = pad_sequences(text_coded, padding='post', maxlen=padding_length)[0,:]\n",
    "    print('Padded input array shape:', input_array.shape)\n",
    "    \n",
    "    ## creat target array from input array\n",
    "    target_array = input_array.copy()\n",
    "    target_array[0:-1] = input_array[1:]\n",
    "    target_array[-1] = input_array[0]\n",
    "    \n",
    "    ## reshaping input and target array to fit stateful training\n",
    "    ## reshaping according to batch_size\n",
    "    input_array = input_array.reshape((batch_size, -1))\n",
    "    target_array = target_array.reshape((batch_size, -1))\n",
    "    ## creating list of batches (link: ...)\n",
    "    x_batches = np.split(input_array, input_array.shape[1] // seq_length, axis=1)\n",
    "    y_batches = np.split(target_array, target_array.shape[1] // seq_length, axis=1)\n",
    "    assert len(x_batches) == len(y_batches)\n",
    "    \n",
    "    ## concatenting list of batches (fit instead of fit generator)\n",
    "    X = np.concatenate(x_batches)\n",
    "    y = np.concatenate(y_batches)\n",
    "    ## additional rank for y array (Keras requirement)\n",
    "    y = y.reshape(y.shape[0], y.shape[1], 1)\n",
    "\n",
    "    print('Input tensor shape:' , X.shape)\n",
    "    print('Target tensor shape:' , y.shape)\n",
    "\n",
    "    return X, y, vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Evaluation of cross sentence LSTMs\n",
    "def cross_sentence_LSTM_eval(model_path, testfile, tokenizer, batch_size=20, seq_length=35):\n",
    "    model = load_model(model_path)\n",
    "    X_test, y_test, _ = txt_to_tensor_cross_sent_LSTM(testfile, tokenizer, batch_size, seq_length)\n",
    "    test_loss = model.evaluate(X_test, y_test, batch_size=batch_size)\n",
    "    print(testfile)\n",
    "    print('Loss: %f\\nPerplexity: %f\\n\\n' % (test_loss, exp(test_loss)))\n",
    "    del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "## stateful 2-layer LSTM for cross sentence modeling\n",
    "def LSTM_stateful_model(vocab_size, embedding_dim, batch_size, seq_length, dropout_rate, LSTM_hidden_size,\n",
    "                        embedding_matrix = 'None'):\n",
    "    model = Sequential()\n",
    "    if embedding_matrix == 'None':\n",
    "        model.add(Embedding(vocab_size, embedding_dim, batch_input_shape=(batch_size, seq_length), mask_zero=True))\n",
    "    else:\n",
    "        model.add(Embedding(vocab_size, embedding_dim, weights=[embedding_matrix], batch_input_shape=(batch_size, seq_length),\n",
    "                  trainable=True, mask_zero=True))\n",
    "    model.add(Dropout(dropout_rate))\n",
    "    model.add(LSTM(LSTM_hidden_size, recurrent_activation='sigmoid', return_sequences=True, stateful=True))\n",
    "    model.add(Dropout(dropout_rate))\n",
    "    model.add(LSTM(LSTM_hidden_size, recurrent_activation='sigmoid', return_sequences=True, stateful=True))\n",
    "    model.add(Dropout(dropout_rate))\n",
    "    model.add(TimeDistributed(Dense(vocab_size, activation='softmax')))\n",
    "    print(model.summary())\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## text generation with cross sentence LSTM\n",
    "def cross_sentence_LSTM_generate(model_path, tokenizer, seed_text='', num_words=100, temperature=1.0,\n",
    "                                 batch_size=20, seq_length=35, random_seed=None):\n",
    "    \n",
    "    ## seed numpy random\n",
    "    np.random.seed(random_seed)\n",
    "\n",
    "    ## load model\n",
    "    model = load_model(model_path)\n",
    "    ## load tokenizer\n",
    "    with open(tokenizer, 'rb') as handle:\n",
    "        tokenizer = pickle.load(handle)\n",
    "    \n",
    "    ## creat id to word mapping dictionary\n",
    "    word_to_id = tokenizer.word_index\n",
    "    id_to_word = {}\n",
    "    for c, i in word_to_id.items():\n",
    "        id_to_word[i] = c\n",
    "    ## add key 0 to dictionary if does not exist\n",
    "    if 0 not in id_to_word:\n",
    "        id_to_word[0] = '<mask>'\n",
    "    \n",
    "    ## coding seed text\n",
    "    sentence = [word_to_id[word] for word in seed_text.split()]\n",
    "\n",
    "    for i in range(num_words):\n",
    "        ## padding for length seq_length\n",
    "        sentence_padded = pad_sequences([sentence], maxlen=seq_length)\n",
    "        ## padding for length seq_length*batch_size, so it will be all zeros in later rows\n",
    "        sentence_padded_postzero = pad_sequences(sentence_padded, maxlen=seq_length*batch_size, padding='post')\n",
    "        ## we arrange so that in each batch only the first line is valuable, the others are zero\n",
    "        sentence_array = np.reshape(sentence_padded_postzero, (-1, seq_length))\n",
    "        ## prediction (we examine the last value of the first row)\n",
    "        preds = model.predict(sentence_array, batch_size=batch_size)[0,-1]\n",
    "        ## we sample the distribution\n",
    "        next_index = sample_pred_simple(preds)\n",
    "        ## the sampled code is translated into words\n",
    "        next_word = id_to_word[next_index]\n",
    "        ## the next input word will be the current word (stateful network saves the history!)\n",
    "        sentence = [next_index]\n",
    "        ## put the next word to output\n",
    "        sys.stdout.write((next_word if next_word != '<eos>' else '\\n') + ' ')\n",
    "        sys.stdout.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "## training function for cross sentence LSTM\n",
    "def cross_sentence_LSTM_train(train_txt, valid_txt, eval_txt, tokenizer, optimizer, callbacks, pretrained_embedding = 'None',\n",
    "                              batch_size = 20, epochs = 100, dropout_rate = 0.5, embedding_dim = 650, LSTM_hidden_size = 650,\n",
    "                              seq_length = 35):\n",
    "\n",
    "    ## extract train txt filename\n",
    "    train_txt_fn = path.splitext(path.basename(train_txt))[0]\n",
    "    ## Creating tensors\n",
    "    X, y, vocab_size = txt_to_tensor_cross_sent_LSTM(train_txt, tokenizer, batch_size, seq_length)\n",
    "    X_valid, y_valid, _ = txt_to_tensor_cross_sent_LSTM(valid_txt, tokenizer, batch_size, seq_length)\n",
    "    ## define model\n",
    "    if pretrained_embedding == 'None':\n",
    "        model = LSTM_stateful_model(vocab_size, embedding_dim, batch_size, seq_length, dropout_rate, LSTM_hidden_size)\n",
    "    else:\n",
    "        ## creating pretrained embedding matrix\n",
    "        embedding_matrix = create_embedding_matrix(pretrained_embedding, tokenizer, embedding_dim)\n",
    "        model = LSTM_stateful_model(vocab_size, embedding_dim, batch_size, seq_length, dropout_rate, LSTM_hidden_size,\n",
    "                                    embedding_matrix = embedding_matrix)\n",
    "    ## compile model\n",
    "    model.compile(loss='sparse_categorical_crossentropy', optimizer=optimizer)\n",
    "    ## extract pretrained embedding filename\n",
    "    pretrained_embedding_fn = path.splitext(path.basename(pretrained_embedding))[0]\n",
    "    ## compile model name\n",
    "    model_fn = 'model/cs-LSTM_{0}_BS-{1}_EMB-{2}.h5'.format(train_txt_fn, batch_size, pretrained_embedding_fn)\n",
    "    ## append model saving to callbacks\n",
    "    callbacks.append(ModelCheckpoint(model_fn, monitor='val_loss', save_best_only=True))\n",
    "    ## fit model\n",
    "    model.fit(X, y, epochs=epochs, callbacks=callbacks, validation_data=(X_valid, y_valid), batch_size=batch_size, shuffle=False)\n",
    "    ## evaluation\n",
    "    cross_sentence_LSTM_eval(model_fn, valid_txt, tokenizer, batch_size, seq_length)\n",
    "    cross_sentence_LSTM_eval(model_fn, eval_txt, tokenizer, batch_size, seq_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## global variables\n",
    "batch_size = 20\n",
    "epochs = 100\n",
    "seq_length = 35\n",
    "LSTM_hidden_size = 650\n",
    "embedding_dim = 650\n",
    "dropout_rate = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "## training and validation data\n",
    "train_txt='data/ptb/ptb_train.txt'\n",
    "valid_txt='data/ptb/ptb_valid.txt'\n",
    "eval_txt='data/ptb/ptb_test.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Sample of original txt:\n",
      "\n",
      "  aer banknote berlitz calloway centrust cluett fromstein gitano guterman hydro-quebec ipo kia memotec mlx nahb punts rake regatta rubens sim snack-food ssangyong swapo wachter \n",
      " pierre <unk> N years old will join the board as a nonexecutive director nov. N \n",
      " mr. <unk> is chairman of <unk> n.v. the d\n",
      "\n",
      "Sample of processed txt:\n",
      "\n",
      "  aer banknote berlitz calloway centrust cluett fromstein gitano guterman hydro-quebec ipo kia memotec mlx nahb punts rake regatta rubens sim snack-food ssangyong swapo wachter  <eos>  pierre <unk> N years old will join the board as a nonexecutive director nov. N  <eos>  mr. <unk> is chairman of <unk\n",
      "\n",
      "Total tokens in text: 929589\n",
      "Unique tokens in text: 10000\n"
     ]
    }
   ],
   "source": [
    "## tokenize train text\n",
    "tokenize_txt_cross_sentence(train_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "## tokenizer\n",
    "tokenizer='tokenizer/cs-ptb_train.pickle'\n",
    "\n",
    "## optimizer\n",
    "optimizer = optimizers.SGD(lr=1.0, momentum=0.9)\n",
    "\n",
    "## Create callback for early stopping on validation loss\n",
    "callbacks = [EarlyStopping(monitor='val_loss', patience=3), \n",
    "             ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=0, min_lr=0.001, verbose=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Sample of original txt:\n",
      "\n",
      "  aer banknote berlitz calloway centrust cluett fromstein gitano guterman hydro-quebec ipo kia memotec mlx nahb punts rake regatta rubens sim snack-food ssangyong swapo wachter \n",
      " pierre <unk> N years old will join the board as a nonexecutive director nov. N \n",
      " mr. <unk> is chairman of <unk> n.v. the d\n",
      "\n",
      "Sample of processed txt:\n",
      "\n",
      "  aer banknote berlitz calloway centrust cluett fromstein gitano guterman hydro-quebec ipo kia memotec mlx nahb punts rake regatta rubens sim snack-food ssangyong swapo wachter  <eos>  pierre <unk> N years old will join the board as a nonexecutive director nov. N  <eos>  mr. <unk> is chairman of <unk\n",
      "\n",
      "Total tokens in text: 929589\n",
      "Unique tokens in text: 10000\n",
      "Vocab size: 10001\n",
      "Coded text length: 929589\n",
      "Padded input array shape: (929600,)\n",
      "Input tensor shape: (26560, 35)\n",
      "Target tensor shape: (26560, 35, 1)\n",
      "\n",
      "Sample of original txt:\n",
      "\n",
      "  consumers may want to move their telephones a little closer to the tv set \n",
      " <unk> <unk> watching abc 's monday night football can now vote during <unk> for the greatest play in N years from among four or five <unk> <unk> \n",
      " two weeks ago viewers of several nbc <unk> consumer segments started calling\n",
      "\n",
      "Sample of processed txt:\n",
      "\n",
      "  consumers may want to move their telephones a little closer to the tv set  <eos>  <unk> <unk> watching abc 's monday night football can now vote during <unk> for the greatest play in N years from among four or five <unk> <unk>  <eos>  two weeks ago viewers of several nbc <unk> consumer segments sta\n",
      "\n",
      "Total tokens in text: 73760\n",
      "Unique tokens in text: 6022\n",
      "Vocab size: 10001\n",
      "Coded text length: 73760\n",
      "Padded input array shape: (74200,)\n",
      "Input tensor shape: (2120, 35)\n",
      "Target tensor shape: (2120, 35, 1)\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (20, 35, 650)             6500650   \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (20, 35, 650)             0         \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (20, 35, 650)             3382600   \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (20, 35, 650)             0         \n",
      "_________________________________________________________________\n",
      "lstm_4 (LSTM)                (20, 35, 650)             3382600   \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (20, 35, 650)             0         \n",
      "_________________________________________________________________\n",
      "time_distributed_2 (TimeDist (20, 35, 10001)           6510651   \n",
      "=================================================================\n",
      "Total params: 19,776,501\n",
      "Trainable params: 19,776,501\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 26560 samples, validate on 2120 samples\n",
      "Epoch 1/100\n",
      "26560/26560 [==============================] - 179s 7ms/step - loss: 6.9043 - val_loss: 6.4516\n",
      "Epoch 2/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 6.2954 - val_loss: 5.6908\n",
      "Epoch 3/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 5.4455 - val_loss: 5.2066\n",
      "Epoch 4/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 5.2242 - val_loss: 5.0793\n",
      "Epoch 5/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 5.0929 - val_loss: 4.9776\n",
      "Epoch 6/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.9877 - val_loss: 4.9041\n",
      "Epoch 7/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.8999 - val_loss: 4.8418\n",
      "Epoch 8/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.8220 - val_loss: 4.7885\n",
      "Epoch 9/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.7525 - val_loss: 4.7502\n",
      "Epoch 10/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.6907 - val_loss: 4.7084\n",
      "Epoch 11/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.6351 - val_loss: 4.6768\n",
      "Epoch 12/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.5841 - val_loss: 4.6513\n",
      "Epoch 13/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.5353 - val_loss: 4.6275\n",
      "Epoch 14/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.4939 - val_loss: 4.6108\n",
      "Epoch 15/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.4517 - val_loss: 4.5896\n",
      "Epoch 16/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.4156 - val_loss: 4.5729\n",
      "Epoch 17/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.3826 - val_loss: 4.5638\n",
      "Epoch 18/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.3513 - val_loss: 4.5537\n",
      "Epoch 19/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.3198 - val_loss: 4.5460\n",
      "Epoch 20/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.2898 - val_loss: 4.5415\n",
      "Epoch 21/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.2616 - val_loss: 4.5315\n",
      "Epoch 22/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.2358 - val_loss: 4.5308\n",
      "Epoch 23/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.2132 - val_loss: 4.5243\n",
      "Epoch 24/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.1889 - val_loss: 4.5192\n",
      "Epoch 25/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.1674 - val_loss: 4.5148\n",
      "Epoch 26/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.1475 - val_loss: 4.5133\n",
      "Epoch 27/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.1288 - val_loss: 4.5142\n",
      "\n",
      "Epoch 00027: ReduceLROnPlateau reducing learning rate to 0.5.\n",
      "Epoch 28/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 4.0256 - val_loss: 4.4922\n",
      "Epoch 29/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.9852 - val_loss: 4.4909\n",
      "Epoch 30/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.9578 - val_loss: 4.4847\n",
      "Epoch 31/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.9381 - val_loss: 4.4846\n",
      "Epoch 32/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.9218 - val_loss: 4.4884\n",
      "\n",
      "Epoch 00032: ReduceLROnPlateau reducing learning rate to 0.25.\n",
      "Epoch 33/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.8648 - val_loss: 4.4790\n",
      "Epoch 34/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.8396 - val_loss: 4.4785\n",
      "Epoch 35/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.8249 - val_loss: 4.4786\n",
      "\n",
      "Epoch 00035: ReduceLROnPlateau reducing learning rate to 0.125.\n",
      "Epoch 36/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7928 - val_loss: 4.4771\n",
      "Epoch 37/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7790 - val_loss: 4.4749\n",
      "Epoch 38/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7716 - val_loss: 4.4737\n",
      "Epoch 39/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7624 - val_loss: 4.4750\n",
      "\n",
      "Epoch 00039: ReduceLROnPlateau reducing learning rate to 0.0625.\n",
      "Epoch 40/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7470 - val_loss: 4.4736\n",
      "\n",
      "Epoch 00040: ReduceLROnPlateau reducing learning rate to 0.03125.\n",
      "Epoch 41/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7364 - val_loss: 4.4710\n",
      "Epoch 42/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7322 - val_loss: 4.4712\n",
      "\n",
      "Epoch 00042: ReduceLROnPlateau reducing learning rate to 0.015625.\n",
      "Epoch 43/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7273 - val_loss: 4.4691\n",
      "Epoch 44/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7260 - val_loss: 4.4696\n",
      "\n",
      "Epoch 00044: ReduceLROnPlateau reducing learning rate to 0.0078125.\n",
      "Epoch 45/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7238 - val_loss: 4.4682\n",
      "Epoch 46/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7229 - val_loss: 4.4683\n",
      "\n",
      "Epoch 00046: ReduceLROnPlateau reducing learning rate to 0.00390625.\n",
      "Epoch 47/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7198 - val_loss: 4.4679\n",
      "Epoch 48/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7207 - val_loss: 4.4676\n",
      "Epoch 49/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7189 - val_loss: 4.4678\n",
      "\n",
      "Epoch 00049: ReduceLROnPlateau reducing learning rate to 0.001953125.\n",
      "Epoch 50/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7190 - val_loss: 4.4675\n",
      "Epoch 51/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7180 - val_loss: 4.4676\n",
      "\n",
      "Epoch 00051: ReduceLROnPlateau reducing learning rate to 0.001.\n",
      "Epoch 52/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7175 - val_loss: 4.4674\n",
      "\n",
      "Epoch 00052: ReduceLROnPlateau reducing learning rate to 0.001.\n",
      "Epoch 53/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7189 - val_loss: 4.4672\n",
      "Epoch 54/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7164 - val_loss: 4.4673\n",
      "\n",
      "Epoch 00054: ReduceLROnPlateau reducing learning rate to 0.001.\n",
      "Epoch 55/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7178 - val_loss: 4.4673\n",
      "\n",
      "Epoch 00055: ReduceLROnPlateau reducing learning rate to 0.001.\n",
      "Epoch 56/100\n",
      "26560/26560 [==============================] - 180s 7ms/step - loss: 3.7160 - val_loss: 4.4673\n",
      "\n",
      "Epoch 00056: ReduceLROnPlateau reducing learning rate to 0.001.\n",
      "\n",
      "Sample of original txt:\n",
      "\n",
      "  consumers may want to move their telephones a little closer to the tv set \n",
      " <unk> <unk> watching abc 's monday night football can now vote during <unk> for the greatest play in N years from among four or five <unk> <unk> \n",
      " two weeks ago viewers of several nbc <unk> consumer segments started calling\n",
      "\n",
      "Sample of processed txt:\n",
      "\n",
      "  consumers may want to move their telephones a little closer to the tv set  <eos>  <unk> <unk> watching abc 's monday night football can now vote during <unk> for the greatest play in N years from among four or five <unk> <unk>  <eos>  two weeks ago viewers of several nbc <unk> consumer segments sta\n",
      "\n",
      "Total tokens in text: 73760\n",
      "Unique tokens in text: 6022\n",
      "Vocab size: 10001\n",
      "Coded text length: 73760\n",
      "Padded input array shape: (74200,)\n",
      "Input tensor shape: (2120, 35)\n",
      "Target tensor shape: (2120, 35, 1)\n",
      "2120/2120 [==============================] - 6s 3ms/step\n",
      "data/ptb/ptb_valid.txt\n",
      "Loss: 4.465162\n",
      "Perplexity: 86.935071\n",
      "\n",
      "\n",
      "\n",
      "Sample of original txt:\n",
      "\n",
      "  no it was n't black monday \n",
      " but while the new york stock exchange did n't fall apart friday as the dow jones industrial average plunged N points most of it in the final hour it barely managed to stay this side of chaos \n",
      " some circuit breakers installed after the october N crash failed their first \n",
      "\n",
      "Sample of processed txt:\n",
      "\n",
      "  no it was n't black monday  <eos>  but while the new york stock exchange did n't fall apart friday as the dow jones industrial average plunged N points most of it in the final hour it barely managed to stay this side of chaos  <eos>  some circuit breakers installed after the october N crash failed \n",
      "\n",
      "Total tokens in text: 82430\n",
      "Unique tokens in text: 6049\n",
      "Vocab size: 10001\n",
      "Coded text length: 82430\n",
      "Padded input array shape: (82600,)\n",
      "Input tensor shape: (2360, 35)\n",
      "Target tensor shape: (2360, 35, 1)\n",
      "2360/2360 [==============================] - 6s 3ms/step\n",
      "data/ptb/ptb_test.txt\n",
      "Loss: 4.414206\n",
      "Perplexity: 82.616243\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cross_sentence_LSTM_train(train_txt, valid_txt, eval_txt, tokenizer, optimizer, callbacks, batch_size=batch_size, \n",
    "                          epochs=epochs, dropout_rate=dropout_rate, embedding_dim=embedding_dim, \n",
    "                          LSTM_hidden_size=LSTM_hidden_size, seq_length=seq_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}