2c. Using Deep (Stacked) LSTM.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "colab": {
      "name": "03. DLSTM v4.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/amfakh/LSTM-Movie-Review/blob/master/2c.%20Using%20Deep%20(Stacked)%20LSTM.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a0RAuwjiBIAv",
        "colab_type": "text"
      },
      "source": [
        "## Force Tensorflow version 1.13.1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YcQfl2Xw3wj_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 746
        },
        "outputId": "3981dc24-37d9-40f2-b174-8e3fa5303ae8"
      },
      "source": [
        "!pip install tensorflow==1.13.1\n",
        "!pip install -qq -U cufflinks\n",
        "!pip install matplotlib==3.1.0\n",
        "import tensorflow as tf\n",
        "tf.logging.set_verbosity(tf.logging.ERROR)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tensorflow==1.13.1 in /usr/local/lib/python3.6/dist-packages (1.13.1)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.17.5)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (3.10.0)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (0.8.1)\n",
            "Requirement already satisfied: tensorflow-estimator<1.14.0rc0,>=1.13.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.13.0)\n",
            "Requirement already satisfied: tensorboard<1.14.0,>=1.13.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.13.1)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.12.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.1.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.1.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (0.33.6)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.15.0)\n",
            "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (0.2.2)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.0.8)\n",
            "Requirement already satisfied: absl-py>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (0.9.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow==1.13.1) (42.0.2)\n",
            "Requirement already satisfied: mock>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-estimator<1.14.0rc0,>=1.13.0->tensorflow==1.13.1) (3.0.5)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow==1.13.1) (0.16.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow==1.13.1) (3.1.1)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow==1.13.1) (2.8.0)\n",
            "Requirement already satisfied: matplotlib==3.1.0 in /usr/local/lib/python3.6/dist-packages (3.1.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.1.0) (2.4.6)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.1.0) (0.10.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.1.0) (2.6.1)\n",
            "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.1.0) (1.17.5)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.1.0) (1.1.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from cycler>=0.10->matplotlib==3.1.0) (1.12.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from kiwisolver>=1.0.1->matplotlib==3.1.0) (42.0.2)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EUdDGKLNhH7m",
        "colab_type": "text"
      },
      "source": [
        "## Access Google Drive files"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "danJ2_x2WArZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c70352f3-23ac-4dae-baab-04a7448179e4"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FebLd3UKY6bp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "1272ff02-2994-4814-a78d-2c4ad4713ea1"
      },
      "source": [
        "%cd /content/drive/My\\ Drive/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q2TTdQzHVxtY",
        "colab_type": "text"
      },
      "source": [
        "# Read all the Files"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v8R-ttCYh25P",
        "colab_type": "text"
      },
      "source": [
        "## Read the dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "B5dzs6P8BFTi",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SKGJfAyLkaH8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.read_csv(\"df.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B_SWTGHmjLA9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "da3e5052-aa0d-4732-a5d1-21760b52b3f8"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>rating</th>\n",
              "      <th>title</th>\n",
              "      <th>review</th>\n",
              "      <th>number</th>\n",
              "      <th>cleaned</th>\n",
              "      <th>sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>10/10</td>\n",
              "      <td>Unlike anything ever done in the history of ci...</td>\n",
              "      <td>This movie is the beginning of the culmination...</td>\n",
              "      <td>10.0</td>\n",
              "      <td>beginning culmination masterfully woven cinema...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>10/10</td>\n",
              "      <td>This movie will blow your mind and break your ...</td>\n",
              "      <td>Over the past decade, Marvel has earned itself...</td>\n",
              "      <td>10.0</td>\n",
              "      <td>past decade earned benefit doubt consistently ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>10/10</td>\n",
              "      <td>Way better than endgame</td>\n",
              "      <td>This film is way better than endgame!\\nThe act...</td>\n",
              "      <td>10.0</td>\n",
              "      <td>way better action better writing better dialog...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>10/10</td>\n",
              "      <td>A Summer Film That IS Even Better Than The Hype</td>\n",
              "      <td>Summer movies often hype themselves as spectac...</td>\n",
              "      <td>10.0</td>\n",
              "      <td>summer often hype spectacular event missed ad ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>9/10</td>\n",
              "      <td>Excellent Film</td>\n",
              "      <td>I was amazed to see so many negative reviews; ...</td>\n",
              "      <td>9.0</td>\n",
              "      <td>amazed negative impossible please hour long co...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0  ... sentiment\n",
              "0           0  ...         1\n",
              "1           1  ...         1\n",
              "2           2  ...         1\n",
              "3           3  ...         1\n",
              "4           4  ...         1\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RfVnOiW7Vxuw",
        "colab_type": "text"
      },
      "source": [
        "# Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rDWlhG7riIej",
        "colab_type": "text"
      },
      "source": [
        "## Split dataset into datatrain and datatest"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kKZ_qtURVxuy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n",
        "# for sent in df:\n",
        "train_review = df['cleaned'].values\n",
        "y_train = df['sentiment'].values\n",
        "\n",
        "    # train_review, test_review, y_train, y_test = train_test_split(\n",
        "    #     review, y, test_size=0.3, shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7_V1qN1DlR6C",
        "colab_type": "text"
      },
      "source": [
        "## Convert text to Bag of Words"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wnxXFsJOj75b",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "09dd1cb1-c9b9-42fb-9382-894919d62f72"
      },
      "source": [
        "from keras.preprocessing.text import Tokenizer\n",
        "\n",
        "# Make sure the data type is str\n",
        "review_train = train_review.astype(str)\n",
        "# review_test = test_review.astype(str)\n",
        "\n",
        "tokenizer = Tokenizer(char_level=False)\n",
        "tokenizer.fit_on_texts(review_train)\n",
        "text_train = tokenizer.texts_to_sequences(review_train)\n",
        "# text_test = tokenizer.texts_to_sequences(review_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8RKaFdrumRU8",
        "colab_type": "text"
      },
      "source": [
        "### (Additional) Declare the vocab size to maximum"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MeDzKUe2mPnZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "2a43c52e-5d3d-47cf-d201-da95bc46a4e0"
      },
      "source": [
        "max_words = len(tokenizer.word_index) + 1  \n",
        "# Adding 1 because of reserved 0 index\n",
        "print('%s unique words.' % max_words)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "37657 unique words.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DqSYqlz9ma2J",
        "colab_type": "text"
      },
      "source": [
        "### (Additional) Declare maximum length to maximum"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sdm00yw7Vxvh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "3ef01242-a660-4a80-99fc-5103824f4fa2"
      },
      "source": [
        "text_len = [len(r) for r in text_train]\n",
        "print(\"average length: %0.1f\" % np.mean(text_len))\n",
        "print(\"max length: %d\" % max(text_len))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "average length: 75.4\n",
            "max length: 899\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xRnRzsuJVxvl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# plt.hist(text_len, bins=500);"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H7ud35jqtATi",
        "colab_type": "text"
      },
      "source": [
        "## Declare max_words and max_len"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AUIFIA_KVxvO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Number of words to consider as features\n",
        "# max_words = 50000\n",
        "# Cut texts after this number of words (among top max_features most common words)\n",
        "max_len = 500"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZhpgG68KmjPc",
        "colab_type": "text"
      },
      "source": [
        "## Pad Sequences"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_QuVFfpS8TCo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# tokenizer.word_index"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4exEzKqvVxvq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "037513bc-87d9-470a-acb0-455fbaa0b468"
      },
      "source": [
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "# max_length = max(text_len)\n",
        "\n",
        "# pad sequences with 0s\n",
        "x = pad_sequences(text_train, maxlen=max_len)\n",
        "# x_test = pad_sequences(text_test, maxlen=max_len)\n",
        "print('Shape of data tensor:', x.shape)\n",
        "# print('Shape of data test tensor:', x_test.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape of data tensor: (25136, 500)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IAAG8Xuint1U",
        "colab_type": "text"
      },
      "source": [
        "### Check the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1gTZu3xAVxvs",
        "colab_type": "code",
        "cellView": "both",
        "colab": {}
      },
      "source": [
        "# print(review_train[4])\n",
        "# print(x_train[4])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "avB7woI-nYFq",
        "colab_type": "code",
        "cellView": "both",
        "colab": {}
      },
      "source": [
        "# print(y_train[0])\n",
        "# print(y_test[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uJzDy6xEn-FK",
        "colab_type": "text"
      },
      "source": [
        "Convert label to float"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EN4W32bErzB8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = np.asarray(y_train).astype('float32').reshape((-1,1))\n",
        "# y_test = np.asarray(y_test).astype('float32').reshape((-1,1))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-aYup7IgoCah",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "4c7ebf84-80d2-41d4-9970-424edfaaaba8"
      },
      "source": [
        "print(y_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1 1 1 ... 1 0 0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IYYmmalwOsVt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# from keras import utils\n",
        "# num_classes = 2\n",
        "\n",
        "# y_train_binary = utils.to_categorical(y_train, num_classes)\n",
        "# y_test_binary = utils.to_categorical(y_test, num_classes)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0nR_WOPVPB72",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# print(y_train_binary)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g70IGUjUBxxb",
        "colab_type": "text"
      },
      "source": [
        "# Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dAOsnygio29e",
        "colab_type": "text"
      },
      "source": [
        "## Import stuff"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "906eUoh_o1zm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "82cf46f7-c3d3-476a-cefc-6843ee59c00b"
      },
      "source": [
        "from tensorflow.python.keras.layers import Dropout, Dense, Embedding\n",
        "from tensorflow.python.keras.layers import SpatialDropout1D, GlobalMaxPool1D\n",
        "from tensorflow.python.keras.layers import LSTM, Input, Bidirectional, GRU\n",
        "\n",
        "print(tf.__version__)\n",
        "print(tf.keras.__version__)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.13.1\n",
            "2.2.4-tf\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gF9_O4lIVxvw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "input_dim = x.shape[1]  # Number of features\n",
        "embedding_dim = 128\n",
        "learning_rate = 2e-5\n",
        "epochs = 500\n",
        "decay_rate = 1e-3 # learning_rate / epochs\n",
        "\n",
        "def make_model(batch_size=None):\n",
        "  source = Input(shape=(max_len,), name='Input')\n",
        "  embedding = Embedding(input_dim=max_words,\n",
        "                        output_dim=embedding_dim,\n",
        "                        input_length=max_len)(source)\n",
        "  # spatial = SpatialDropout1D(0.4)(embedding)\n",
        "  lstm = LSTM(128, \n",
        "              # activation='softsign',\n",
        "              dropout=0.3,\n",
        "              recurrent_dropout=0.3\n",
        "              # ,use_bias=True\n",
        "              ,return_sequences=True\n",
        "              )(embedding)\n",
        "  lstm = LSTM(128, \n",
        "            # activation='softsign',\n",
        "            dropout=0.3,\n",
        "            recurrent_dropout=0.3\n",
        "            # ,use_bias=True\n",
        "            ,return_sequences=True\n",
        "            )(lstm)\n",
        "  lstm = LSTM(128, \n",
        "            # activation='softsign',\n",
        "            dropout=0.3,\n",
        "            recurrent_dropout=0.3\n",
        "            # ,use_bias=True\n",
        "            # ,return_sequences=True\n",
        "            )(lstm)\n",
        "  # pool = GlobalMaxPool1D()(lstm)\n",
        "  # dense = Dense(24)(lstm)\n",
        "  # drop = Dropout(0.4)(dense)\n",
        "\n",
        "  # lstm = Bidirectional(LSTM(196, activation='softsign',\n",
        "  #           dropout=0.7,\n",
        "  #           recurrent_dropout=0.7,\n",
        "  #           use_bias=True\n",
        "  #           # ,return_sequences=True\n",
        "  #           ))(drop)\n",
        "  # dense = Dense(50)(lstm)\n",
        "  # drop = Dropout(0.7)(dense)\n",
        "\n",
        "  predict = Dense(1, activation='sigmoid')(lstm)\n",
        "\n",
        "  model = tf.keras.Model(inputs=[source], outputs=[predict])\n",
        "\n",
        "  rmsprop = tf.train.RMSPropOptimizer(learning_rate=learning_rate, decay=decay_rate)\n",
        "  model.compile(\n",
        "      optimizer=rmsprop,\n",
        "      loss='binary_crossentropy',\n",
        "      metrics=['accuracy']\n",
        "  )\n",
        "  return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EenkZdsjvHsF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "\n",
        "TPU_WORKER = 'grpc://' + os.environ['COLAB_TPU_ADDR']\n",
        "tf.logging.set_verbosity(tf.logging.INFO)\n",
        "\n",
        "# tpu_model.summary()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Rnwt-lEyKTI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "seed = 42\n",
        "np.random.seed(seed)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D1ChsH4VVxv7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.style.use('ggplot')\n",
        "\n",
        "def plot_history(history):\n",
        "    acc = history.history['acc']\n",
        "    val_acc = history.history['val_acc']\n",
        "    loss = history.history['loss']\n",
        "    val_loss = history.history['val_loss']\n",
        "    x = range(1, len(acc) + 1)\n",
        "\n",
        "    plt.figure(figsize=(15, 5))\n",
        "    plt.subplot(1, 2, 1)\n",
        "    plt.plot(x, acc, 'b', label='Train acc')\n",
        "    plt.plot(x, val_acc, 'r', label='Validation acc')\n",
        "    plt.title('Train and validation accuracy')\n",
        "    plt.legend()\n",
        "    plt.subplot(1, 2, 2)\n",
        "    plt.plot(x, loss, 'b', label='Train loss')\n",
        "    plt.plot(x, val_loss, 'r', label='Validation loss')\n",
        "    plt.title('Train and validation loss')\n",
        "    plt.legend()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dUDhBL7kv_9L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import confusion_matrix, classification_report, roc_curve,roc_auc_score\n",
        "\n",
        "import seaborn as sns\n",
        "\n",
        "\n",
        "\n",
        "# sns.set()\n",
        "# sns.heatmap(confusion.T, square=True, annot=True, fmt='d', cbar=False,\n",
        "#             xticklabels=np.unique(y_pred),\n",
        "#             yticklabels=np.unique(y_pred))\n",
        "# plt.xlabel('true label')\n",
        "# plt.ylabel('predicted label')\n",
        "# plt.title('Confusion Matrix Prediction')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t0h77kZoUcym",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "bf23df92-923b-43e5-a986-4aa79bcd0f01"
      },
      "source": [
        "from sklearn.model_selection import StratifiedKFold\n",
        "from sklearn.metrics import mean_squared_error\n",
        "import time, math\n",
        "from keras.callbacks import ReduceLROnPlateau, EarlyStopping\n",
        "\n",
        "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed)\n",
        "es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=7)\n",
        "\n",
        "tr_acc_array = []\n",
        "tr_loss_array = []\n",
        "te_acc_array = []\n",
        "te_loss_array = []\n",
        "time_array = []\n",
        "rmse_array = []\n",
        "\n",
        "n=0\n",
        "\n",
        "\n",
        "for train, test in kfold.split(x, y):\n",
        "  n+=1\n",
        "  print(\"--- Fold %d ---\" % (n))\n",
        "  tf.keras.backend.clear_session()\n",
        "  training_model = make_model(batch_size = 128)\n",
        "  # training_model.summary()\n",
        "\n",
        "  tpu_model = tf.contrib.tpu.keras_to_tpu_model(\n",
        "    training_model,\n",
        "    strategy=tf.contrib.tpu.TPUDistributionStrategy(\n",
        "        tf.contrib.cluster_resolver.TPUClusterResolver(TPU_WORKER)))\n",
        "\n",
        "  start_time = time.time()\n",
        "  history = tpu_model.fit(x[train], y[train]\n",
        "                    ,epochs=epochs, verbose=1 \n",
        "                    ,validation_split=0.2\n",
        "                    ,batch_size=128 * 8\n",
        "                    ,validation_data=(x[test], y[test])\n",
        "                    ,callbacks=[es]\n",
        "                   )\n",
        "  print(\"--- %s seconds ---\" % (time.time() - start_time))\n",
        "  time_array.append(time.time() - start_time)\n",
        "\n",
        "  model = tpu_model.sync_to_cpu()\n",
        "\n",
        "  tr_loss, tr_accuracy = model.evaluate(x[train], y[train], verbose=1)\n",
        "  te_loss, te_accuracy = model.evaluate(x[test], y[test], verbose=1)\n",
        "  print(\"Train Accuracy: {:.4f}\".format(tr_accuracy))\n",
        "  tr_acc_array.append(tr_accuracy)\n",
        "  print(\"Train Loss: {:.4f}\".format(tr_loss))\n",
        "  tr_loss_array.append(tr_loss)\n",
        "  print(\"Validation Accuracy:  {:.4f}\".format(te_accuracy))\n",
        "  te_acc_array.append(te_accuracy)\n",
        "  print(\"Validation Loss: {:.4f}\".format(te_loss))\n",
        "  te_loss_array.append(te_loss)\n",
        "\n",
        "  pred = model.predict(x[test], batch_size=1000, verbose = 1)\n",
        "  y_pred = y_pred = (pred > 0.5)\n",
        "\n",
        "  print(confusion_matrix(y[test], y_pred))\n",
        "  # sns.set()\n",
        "  # sns.heatmap(confusion.T, square=True, annot=True, fmt='d', cbar=False,\n",
        "  #             xticklabels=np.unique(y_pred),\n",
        "  #             yticklabels=np.unique(y_pred))\n",
        "  # plt.xlabel('true label')\n",
        "  # plt.ylabel('predicted label')\n",
        "  # plt.title('Confusion Matrix Prediction')\n",
        "\n",
        "\n",
        "  print(classification_report(y[test], y_pred))\n",
        "  \n",
        "  # calculate root mean squared error\n",
        "  rmse = math.sqrt(mean_squared_error(y[test], y_pred))\n",
        "  print('RMSE: %.4f' % (rmse))\n",
        "  rmse_array.append(rmse)\n",
        "  # testScore = math.sqrt(mean_squared_error(testY[0], testPredict[:,0]))\n",
        "  # print('Test Score: %.2f RMSE' % (testScore))\n",
        "\n",
        "  # auc_score=roc_auc_score(y[test], y_pred)  \n",
        "  # print('AUC: %.2f' % (auc_score))\n",
        "\n",
        "  # cvscores.append(scores[1] * 100)\n",
        "\n",
        "  # plot_history(history)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--- Fold 1 ---\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Colocations handled automatically by placer.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/backend.py:4010: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
            "\n",
            "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n",
            "INFO:tensorflow:Querying Tensorflow master (grpc://10.126.17.194:8470) for TPU system metadata.\n",
            "INFO:tensorflow:Found TPU system:\n",
            "INFO:tensorflow:*** Num TPU Cores: 8\n",
            "INFO:tensorflow:*** Num TPU Workers: 1\n",
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 12028669822908458194)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, 8434022613342494329)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 9483302766372785140)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, 16913650255987291235)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, 9584223271632520063)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 5131928026390797549)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 3800337932879369162)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 2724390565822717472)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 1277980390101900352)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, 15973569101757891556)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 8589934592, 13938748725989702413)\n",
            "WARNING:tensorflow:tpu_model (from tensorflow.contrib.tpu.python.tpu.keras_support) is experimental and may change or be removed at any time, and without warning.\n",
            "Train on 20108 samples, validate on 5028 samples\n",
            "Epoch 1/500\n",
            "INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(128, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(128, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/tpu/python/tpu/keras_support.py:302: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.cast instead.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.cast instead.\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 7.5635740756988525 secs\n",
            "INFO:tensorflow:Setting weights on TPU model.\n",
            "18432/20108 [==========================>...] - ETA: 2s - loss: 0.6922 - acc: 0.6175INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(81,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(81, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(81, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 9.384580373764038 secs\n",
            "19456/20108 [============================>.] - ETA: 1s - loss: 0.6922 - acc: 0.6214INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(128, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(128, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 8.520802974700928 secs\n",
            "INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(116,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(116, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(116, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 10.25054931640625 secs\n",
            "20108/20108 [==============================] - 80s 4ms/sample - loss: 0.6921 - acc: 0.6239 - val_loss: 0.6908 - val_acc: 0.7014\n",
            "Epoch 2/500\n",
            "20108/20108 [==============================] - 13s 642us/sample - loss: 0.6895 - acc: 0.7013 - val_loss: 0.6879 - val_acc: 0.7014\n",
            "Epoch 3/500\n",
            "20108/20108 [==============================] - 13s 643us/sample - loss: 0.6863 - acc: 0.7017 - val_loss: 0.6842 - val_acc: 0.7014\n",
            "Epoch 4/500\n",
            "20108/20108 [==============================] - 13s 638us/sample - loss: 0.6821 - acc: 0.7017 - val_loss: 0.6794 - val_acc: 0.7014\n",
            "Epoch 5/500\n",
            "20108/20108 [==============================] - 13s 630us/sample - loss: 0.6768 - acc: 0.7017 - val_loss: 0.6731 - val_acc: 0.7014\n",
            "Epoch 6/500\n",
            "20108/20108 [==============================] - 13s 644us/sample - loss: 0.6692 - acc: 0.7017 - val_loss: 0.6648 - val_acc: 0.7014\n",
            "Epoch 7/500\n",
            "20108/20108 [==============================] - 13s 645us/sample - loss: 0.6597 - acc: 0.7017 - val_loss: 0.6540 - val_acc: 0.7014\n",
            "Epoch 8/500\n",
            "20108/20108 [==============================] - 13s 640us/sample - loss: 0.6480 - acc: 0.7017 - val_loss: 0.6408 - val_acc: 0.7014\n",
            "Epoch 9/500\n",
            "20108/20108 [==============================] - 13s 629us/sample - loss: 0.6352 - acc: 0.7017 - val_loss: 0.6278 - val_acc: 0.7014\n",
            "Epoch 10/500\n",
            "20108/20108 [==============================] - 13s 623us/sample - loss: 0.6255 - acc: 0.7017 - val_loss: 0.6199 - val_acc: 0.7014\n",
            "Epoch 11/500\n",
            "20108/20108 [==============================] - 13s 637us/sample - loss: 0.6202 - acc: 0.7017 - val_loss: 0.6161 - val_acc: 0.7014\n",
            "Epoch 12/500\n",
            "20108/20108 [==============================] - 13s 624us/sample - loss: 0.6167 - acc: 0.7017 - val_loss: 0.6134 - val_acc: 0.7014\n",
            "Epoch 13/500\n",
            "20108/20108 [==============================] - 13s 635us/sample - loss: 0.6141 - acc: 0.7017 - val_loss: 0.6113 - val_acc: 0.7014\n",
            "Epoch 14/500\n",
            "20108/20108 [==============================] - 13s 642us/sample - loss: 0.6123 - acc: 0.7016 - val_loss: 0.6093 - val_acc: 0.7014\n",
            "Epoch 15/500\n",
            "20108/20108 [==============================] - 13s 637us/sample - loss: 0.6106 - acc: 0.7017 - val_loss: 0.6077 - val_acc: 0.7014\n",
            "Epoch 16/500\n",
            "20108/20108 [==============================] - 13s 623us/sample - loss: 0.6082 - acc: 0.7016 - val_loss: 0.6060 - val_acc: 0.7014\n",
            "Epoch 17/500\n",
            "20108/20108 [==============================] - 12s 620us/sample - loss: 0.6064 - acc: 0.7017 - val_loss: 0.6047 - val_acc: 0.7014\n",
            "Epoch 18/500\n",
            "20108/20108 [==============================] - 13s 642us/sample - loss: 0.6050 - acc: 0.7017 - val_loss: 0.6032 - val_acc: 0.7014\n",
            "Epoch 19/500\n",
            "20108/20108 [==============================] - 13s 640us/sample - loss: 0.6048 - acc: 0.7017 - val_loss: 0.6017 - val_acc: 0.7014\n",
            "Epoch 20/500\n",
            "20108/20108 [==============================] - 13s 649us/sample - loss: 0.6033 - acc: 0.7018 - val_loss: 0.6001 - val_acc: 0.7014\n",
            "Epoch 21/500\n",
            "20108/20108 [==============================] - 13s 634us/sample - loss: 0.6015 - acc: 0.7016 - val_loss: 0.5985 - val_acc: 0.7014\n",
            "Epoch 22/500\n",
            "20108/20108 [==============================] - 13s 642us/sample - loss: 0.6000 - acc: 0.7016 - val_loss: 0.5966 - val_acc: 0.7014\n",
            "Epoch 23/500\n",
            "20108/20108 [==============================] - 13s 634us/sample - loss: 0.5980 - acc: 0.7016 - val_loss: 0.5943 - val_acc: 0.7014\n",
            "Epoch 24/500\n",
            "20108/20108 [==============================] - 13s 635us/sample - loss: 0.5944 - acc: 0.7016 - val_loss: 0.5916 - val_acc: 0.7014\n",
            "Epoch 25/500\n",
            "20108/20108 [==============================] - 13s 636us/sample - loss: 0.5921 - acc: 0.7017 - val_loss: 0.5885 - val_acc: 0.7014\n",
            "Epoch 26/500\n",
            "20108/20108 [==============================] - 13s 633us/sample - loss: 0.5882 - acc: 0.7016 - val_loss: 0.5847 - val_acc: 0.7014\n",
            "Epoch 27/500\n",
            "20108/20108 [==============================] - 12s 614us/sample - loss: 0.5854 - acc: 0.7017 - val_loss: 0.5806 - val_acc: 0.7014\n",
            "Epoch 28/500\n",
            "20108/20108 [==============================] - 13s 625us/sample - loss: 0.5807 - acc: 0.7016 - val_loss: 0.5751 - val_acc: 0.7014\n",
            "Epoch 29/500\n",
            "20108/20108 [==============================] - 13s 631us/sample - loss: 0.5761 - acc: 0.7017 - val_loss: 0.5699 - val_acc: 0.7014\n",
            "Epoch 30/500\n",
            "20108/20108 [==============================] - 13s 639us/sample - loss: 0.5701 - acc: 0.7018 - val_loss: 0.5627 - val_acc: 0.7014\n",
            "Epoch 31/500\n",
            "20108/20108 [==============================] - 13s 630us/sample - loss: 0.5641 - acc: 0.7021 - val_loss: 0.5547 - val_acc: 0.7014\n",
            "Epoch 32/500\n",
            "20108/20108 [==============================] - 13s 645us/sample - loss: 0.5552 - acc: 0.7026 - val_loss: 0.5447 - val_acc: 0.7022\n",
            "Epoch 33/500\n",
            "20108/20108 [==============================] - 13s 627us/sample - loss: 0.5489 - acc: 0.7037 - val_loss: 0.5342 - val_acc: 0.7036\n",
            "Epoch 34/500\n",
            "20108/20108 [==============================] - 13s 632us/sample - loss: 0.5390 - acc: 0.7097 - val_loss: 0.5226 - val_acc: 0.7078\n",
            "Epoch 35/500\n",
            "20108/20108 [==============================] - 13s 628us/sample - loss: 0.5319 - acc: 0.7161 - val_loss: 0.5110 - val_acc: 0.7184\n",
            "Epoch 36/500\n",
            "20108/20108 [==============================] - 13s 650us/sample - loss: 0.5179 - acc: 0.7283 - val_loss: 0.4974 - val_acc: 0.7325\n",
            "Epoch 37/500\n",
            "20108/20108 [==============================] - 12s 609us/sample - loss: 0.5113 - acc: 0.7413 - val_loss: 0.4864 - val_acc: 0.7438\n",
            "Epoch 38/500\n",
            "20108/20108 [==============================] - 13s 631us/sample - loss: 0.4993 - acc: 0.7525 - val_loss: 0.4761 - val_acc: 0.7536\n",
            "Epoch 39/500\n",
            "20108/20108 [==============================] - 13s 633us/sample - loss: 0.4942 - acc: 0.7619 - val_loss: 0.4673 - val_acc: 0.7625\n",
            "Epoch 40/500\n",
            "20108/20108 [==============================] - 13s 626us/sample - loss: 0.4914 - acc: 0.7624 - val_loss: 0.4631 - val_acc: 0.7715\n",
            "Epoch 41/500\n",
            "20108/20108 [==============================] - 13s 634us/sample - loss: 0.4832 - acc: 0.7702 - val_loss: 0.4572 - val_acc: 0.7781\n",
            "Epoch 42/500\n",
            "20108/20108 [==============================] - 13s 625us/sample - loss: 0.4795 - acc: 0.7708 - val_loss: 0.4511 - val_acc: 0.7773\n",
            "Epoch 43/500\n",
            "20108/20108 [==============================] - 13s 632us/sample - loss: 0.4765 - acc: 0.7749 - val_loss: 0.4472 - val_acc: 0.7803\n",
            "Epoch 44/500\n",
            "20108/20108 [==============================] - 13s 632us/sample - loss: 0.4711 - acc: 0.7772 - val_loss: 0.4438 - val_acc: 0.7874\n",
            "Epoch 45/500\n",
            "20108/20108 [==============================] - 13s 636us/sample - loss: 0.4634 - acc: 0.7827 - val_loss: 0.4383 - val_acc: 0.7900\n",
            "Epoch 46/500\n",
            "20108/20108 [==============================] - 13s 632us/sample - loss: 0.4595 - acc: 0.7866 - val_loss: 0.4358 - val_acc: 0.7942\n",
            "Epoch 47/500\n",
            "20108/20108 [==============================] - 13s 623us/sample - loss: 0.4547 - acc: 0.7870 - val_loss: 0.4315 - val_acc: 0.7932\n",
            "Epoch 48/500\n",
            "20108/20108 [==============================] - 13s 625us/sample - loss: 0.4460 - acc: 0.7918 - val_loss: 0.4284 - val_acc: 0.7942\n",
            "Epoch 49/500\n",
            "20108/20108 [==============================] - 13s 633us/sample - loss: 0.4413 - acc: 0.7935 - val_loss: 0.4246 - val_acc: 0.7986\n",
            "Epoch 50/500\n",
            "20108/20108 [==============================] - 13s 651us/sample - loss: 0.4382 - acc: 0.7970 - val_loss: 0.4224 - val_acc: 0.7984\n",
            "Epoch 51/500\n",
            "20108/20108 [==============================] - 13s 624us/sample - loss: 0.4377 - acc: 0.7955 - val_loss: 0.4207 - val_acc: 0.7988\n",
            "Epoch 52/500\n",
            "20108/20108 [==============================] - 13s 638us/sample - loss: 0.4313 - acc: 0.8002 - val_loss: 0.4180 - val_acc: 0.8021\n",
            "Epoch 53/500\n",
            "20108/20108 [==============================] - 13s 629us/sample - loss: 0.4271 - acc: 0.8061 - val_loss: 0.4153 - val_acc: 0.8023\n",
            "Epoch 54/500\n",
            "20108/20108 [==============================] - 13s 641us/sample - loss: 0.4273 - acc: 0.8039 - val_loss: 0.4141 - val_acc: 0.8031\n",
            "Epoch 55/500\n",
            "20108/20108 [==============================] - 13s 639us/sample - loss: 0.4197 - acc: 0.8054 - val_loss: 0.4118 - val_acc: 0.8033\n",
            "Epoch 56/500\n",
            "20108/20108 [==============================] - 13s 636us/sample - loss: 0.4208 - acc: 0.8038 - val_loss: 0.4112 - val_acc: 0.8023\n",
            "Epoch 57/500\n",
            "20108/20108 [==============================] - 13s 633us/sample - loss: 0.4151 - acc: 0.8112 - val_loss: 0.4090 - val_acc: 0.8039\n",
            "Epoch 58/500\n",
            "20108/20108 [==============================] - 13s 634us/sample - loss: 0.4113 - acc: 0.8145 - val_loss: 0.4070 - val_acc: 0.8057\n",
            "Epoch 59/500\n",
            "20108/20108 [==============================] - 13s 637us/sample - loss: 0.4093 - acc: 0.8138 - val_loss: 0.4058 - val_acc: 0.8067\n",
            "Epoch 60/500\n",
            "20108/20108 [==============================] - 13s 648us/sample - loss: 0.4069 - acc: 0.8153 - val_loss: 0.4039 - val_acc: 0.8069\n",
            "Epoch 61/500\n",
            "20108/20108 [==============================] - 13s 644us/sample - loss: 0.4021 - acc: 0.8177 - val_loss: 0.4022 - val_acc: 0.8097\n",
            "Epoch 62/500\n",
            "20108/20108 [==============================] - 12s 617us/sample - loss: 0.4012 - acc: 0.8192 - val_loss: 0.4008 - val_acc: 0.8085\n",
            "Epoch 63/500\n",
            "20108/20108 [==============================] - 12s 620us/sample - loss: 0.3979 - acc: 0.8223 - val_loss: 0.3994 - val_acc: 0.8109\n",
            "Epoch 64/500\n",
            "20108/20108 [==============================] - 13s 630us/sample - loss: 0.3977 - acc: 0.8215 - val_loss: 0.3990 - val_acc: 0.8103\n",
            "Epoch 65/500\n",
            "20108/20108 [==============================] - 13s 643us/sample - loss: 0.3916 - acc: 0.8258 - val_loss: 0.3971 - val_acc: 0.8127\n",
            "Epoch 66/500\n",
            "20108/20108 [==============================] - 13s 630us/sample - loss: 0.3907 - acc: 0.8228 - val_loss: 0.3969 - val_acc: 0.8125\n",
            "Epoch 67/500\n",
            "20108/20108 [==============================] - 13s 625us/sample - loss: 0.3903 - acc: 0.8238 - val_loss: 0.3955 - val_acc: 0.8143\n",
            "Epoch 68/500\n",
            "20108/20108 [==============================] - 13s 631us/sample - loss: 0.3875 - acc: 0.8251 - val_loss: 0.3946 - val_acc: 0.8155\n",
            "Epoch 69/500\n",
            "20108/20108 [==============================] - 13s 625us/sample - loss: 0.3858 - acc: 0.8267 - val_loss: 0.3936 - val_acc: 0.8161\n",
            "Epoch 70/500\n",
            "20108/20108 [==============================] - 13s 649us/sample - loss: 0.3803 - acc: 0.8297 - val_loss: 0.3914 - val_acc: 0.8183\n",
            "Epoch 71/500\n",
            "20108/20108 [==============================] - 13s 630us/sample - loss: 0.3798 - acc: 0.8298 - val_loss: 0.3915 - val_acc: 0.8183\n",
            "Epoch 72/500\n",
            "20108/20108 [==============================] - 13s 628us/sample - loss: 0.3791 - acc: 0.8309 - val_loss: 0.3907 - val_acc: 0.8183\n",
            "Epoch 73/500\n",
            "20108/20108 [==============================] - 13s 629us/sample - loss: 0.3773 - acc: 0.8298 - val_loss: 0.3896 - val_acc: 0.8197\n",
            "Epoch 74/500\n",
            "20108/20108 [==============================] - 13s 635us/sample - loss: 0.3726 - acc: 0.8346 - val_loss: 0.3886 - val_acc: 0.8201\n",
            "Epoch 75/500\n",
            "20108/20108 [==============================] - 13s 635us/sample - loss: 0.3724 - acc: 0.8337 - val_loss: 0.3876 - val_acc: 0.8207\n",
            "Epoch 76/500\n",
            "20108/20108 [==============================] - 13s 638us/sample - loss: 0.3696 - acc: 0.8386 - val_loss: 0.3877 - val_acc: 0.8203\n",
            "Epoch 77/500\n",
            "20108/20108 [==============================] - 12s 619us/sample - loss: 0.3690 - acc: 0.8356 - val_loss: 0.3861 - val_acc: 0.8207\n",
            "Epoch 78/500\n",
            "20108/20108 [==============================] - 13s 629us/sample - loss: 0.3656 - acc: 0.8401 - val_loss: 0.3857 - val_acc: 0.8215\n",
            "Epoch 79/500\n",
            "20108/20108 [==============================] - 13s 632us/sample - loss: 0.3654 - acc: 0.8370 - val_loss: 0.3842 - val_acc: 0.8244\n",
            "Epoch 80/500\n",
            "20108/20108 [==============================] - 13s 628us/sample - loss: 0.3642 - acc: 0.8402 - val_loss: 0.3843 - val_acc: 0.8221\n",
            "Epoch 81/500\n",
            "20108/20108 [==============================] - 13s 639us/sample - loss: 0.3616 - acc: 0.8396 - val_loss: 0.3829 - val_acc: 0.8254\n",
            "Epoch 82/500\n",
            "20108/20108 [==============================] - 13s 630us/sample - loss: 0.3565 - acc: 0.8435 - val_loss: 0.3827 - val_acc: 0.8264\n",
            "Epoch 83/500\n",
            "20108/20108 [==============================] - 13s 626us/sample - loss: 0.3605 - acc: 0.8423 - val_loss: 0.3819 - val_acc: 0.8274\n",
            "Epoch 84/500\n",
            "20108/20108 [==============================] - 13s 631us/sample - loss: 0.3600 - acc: 0.8421 - val_loss: 0.3817 - val_acc: 0.8276\n",
            "Epoch 85/500\n",
            "20108/20108 [==============================] - 13s 639us/sample - loss: 0.3539 - acc: 0.8456 - val_loss: 0.3825 - val_acc: 0.8238\n",
            "Epoch 86/500\n",
            "20108/20108 [==============================] - 12s 616us/sample - loss: 0.3537 - acc: 0.8455 - val_loss: 0.3802 - val_acc: 0.8304\n",
            "Epoch 87/500\n",
            "20108/20108 [==============================] - 13s 624us/sample - loss: 0.3517 - acc: 0.8460 - val_loss: 0.3809 - val_acc: 0.8258\n",
            "Epoch 88/500\n",
            "20108/20108 [==============================] - 13s 642us/sample - loss: 0.3497 - acc: 0.8465 - val_loss: 0.3801 - val_acc: 0.8282\n",
            "Epoch 89/500\n",
            "20108/20108 [==============================] - 13s 627us/sample - loss: 0.3508 - acc: 0.8471 - val_loss: 0.3793 - val_acc: 0.8290\n",
            "Epoch 90/500\n",
            "20108/20108 [==============================] - 13s 650us/sample - loss: 0.3468 - acc: 0.8512 - val_loss: 0.3794 - val_acc: 0.8274\n",
            "Epoch 91/500\n",
            "20108/20108 [==============================] - 13s 632us/sample - loss: 0.3454 - acc: 0.8513 - val_loss: 0.3783 - val_acc: 0.8296\n",
            "Epoch 92/500\n",
            "20108/20108 [==============================] - 13s 631us/sample - loss: 0.3428 - acc: 0.8518 - val_loss: 0.3773 - val_acc: 0.8310\n",
            "Epoch 93/500\n",
            "20108/20108 [==============================] - 13s 629us/sample - loss: 0.3413 - acc: 0.8528 - val_loss: 0.3772 - val_acc: 0.8308\n",
            "Epoch 94/500\n",
            "20108/20108 [==============================] - 13s 638us/sample - loss: 0.3396 - acc: 0.8532 - val_loss: 0.3768 - val_acc: 0.8298\n",
            "Epoch 95/500\n",
            "20108/20108 [==============================] - 13s 634us/sample - loss: 0.3403 - acc: 0.8530 - val_loss: 0.3767 - val_acc: 0.8306\n",
            "Epoch 96/500\n",
            "20108/20108 [==============================] - 13s 623us/sample - loss: 0.3381 - acc: 0.8542 - val_loss: 0.3767 - val_acc: 0.8306\n",
            "Epoch 97/500\n",
            "20108/20108 [==============================] - 13s 634us/sample - loss: 0.3378 - acc: 0.8559 - val_loss: 0.3760 - val_acc: 0.8318\n",
            "Epoch 98/500\n",
            "20108/20108 [==============================] - 13s 637us/sample - loss: 0.3400 - acc: 0.8524 - val_loss: 0.3753 - val_acc: 0.8316\n",
            "Epoch 99/500\n",
            "20108/20108 [==============================] - 13s 630us/sample - loss: 0.3352 - acc: 0.8565 - val_loss: 0.3743 - val_acc: 0.8332\n",
            "Epoch 100/500\n",
            "20108/20108 [==============================] - 12s 619us/sample - loss: 0.3378 - acc: 0.8544 - val_loss: 0.3740 - val_acc: 0.8336\n",
            "Epoch 101/500\n",
            "20108/20108 [==============================] - 13s 626us/sample - loss: 0.3318 - acc: 0.8590 - val_loss: 0.3733 - val_acc: 0.8346\n",
            "Epoch 102/500\n",
            "20108/20108 [==============================] - 13s 634us/sample - loss: 0.3334 - acc: 0.8576 - val_loss: 0.3735 - val_acc: 0.8356\n",
            "Epoch 103/500\n",
            "20108/20108 [==============================] - 13s 627us/sample - loss: 0.3319 - acc: 0.8601 - val_loss: 0.3736 - val_acc: 0.8360\n",
            "Epoch 104/500\n",
            "20108/20108 [==============================] - 13s 647us/sample - loss: 0.3290 - acc: 0.8588 - val_loss: 0.3725 - val_acc: 0.8354\n",
            "Epoch 105/500\n",
            "20108/20108 [==============================] - 13s 635us/sample - loss: 0.3292 - acc: 0.8622 - val_loss: 0.3728 - val_acc: 0.8370\n",
            "Epoch 106/500\n",
            "20108/20108 [==============================] - 13s 634us/sample - loss: 0.3267 - acc: 0.8606 - val_loss: 0.3730 - val_acc: 0.8368\n",
            "Epoch 107/500\n",
            "20108/20108 [==============================] - 12s 618us/sample - loss: 0.3280 - acc: 0.8599 - val_loss: 0.3725 - val_acc: 0.8368\n",
            "Epoch 108/500\n",
            "20108/20108 [==============================] - 13s 648us/sample - loss: 0.3244 - acc: 0.8618 - val_loss: 0.3719 - val_acc: 0.8378\n",
            "Epoch 109/500\n",
            "20108/20108 [==============================] - 13s 633us/sample - loss: 0.3240 - acc: 0.8624 - val_loss: 0.3715 - val_acc: 0.8376\n",
            "Epoch 110/500\n",
            "20108/20108 [==============================] - 13s 639us/sample - loss: 0.3254 - acc: 0.8616 - val_loss: 0.3710 - val_acc: 0.8386\n",
            "Epoch 111/500\n",
            "20108/20108 [==============================] - 13s 623us/sample - loss: 0.3203 - acc: 0.8642 - val_loss: 0.3715 - val_acc: 0.8384\n",
            "Epoch 112/500\n",
            "20108/20108 [==============================] - 13s 639us/sample - loss: 0.3209 - acc: 0.8660 - val_loss: 0.3714 - val_acc: 0.8380\n",
            "Epoch 113/500\n",
            "20108/20108 [==============================] - 13s 632us/sample - loss: 0.3244 - acc: 0.8638 - val_loss: 0.3705 - val_acc: 0.8392\n",
            "Epoch 114/500\n",
            "20108/20108 [==============================] - 13s 646us/sample - loss: 0.3208 - acc: 0.8659 - val_loss: 0.3704 - val_acc: 0.8394\n",
            "Epoch 115/500\n",
            "20108/20108 [==============================] - 12s 618us/sample - loss: 0.3180 - acc: 0.8664 - val_loss: 0.3718 - val_acc: 0.8380\n",
            "Epoch 116/500\n",
            "20108/20108 [==============================] - 13s 629us/sample - loss: 0.3182 - acc: 0.8664 - val_loss: 0.3712 - val_acc: 0.8382\n",
            "Epoch 117/500\n",
            "20108/20108 [==============================] - 13s 636us/sample - loss: 0.3174 - acc: 0.8663 - val_loss: 0.3699 - val_acc: 0.8400\n",
            "Epoch 118/500\n",
            "20108/20108 [==============================] - 13s 641us/sample - loss: 0.3173 - acc: 0.8666 - val_loss: 0.3698 - val_acc: 0.8404\n",
            "Epoch 119/500\n",
            "20108/20108 [==============================] - 13s 636us/sample - loss: 0.3161 - acc: 0.8677 - val_loss: 0.3691 - val_acc: 0.8402\n",
            "Epoch 120/500\n",
            "20108/20108 [==============================] - 13s 628us/sample - loss: 0.3147 - acc: 0.8688 - val_loss: 0.3690 - val_acc: 0.8402\n",
            "Epoch 121/500\n",
            "20108/20108 [==============================] - 13s 624us/sample - loss: 0.3127 - acc: 0.8711 - val_loss: 0.3690 - val_acc: 0.8404\n",
            "Epoch 122/500\n",
            "20108/20108 [==============================] - 13s 633us/sample - loss: 0.3115 - acc: 0.8704 - val_loss: 0.3696 - val_acc: 0.8412\n",
            "Epoch 123/500\n",
            "20108/20108 [==============================] - 13s 632us/sample - loss: 0.3088 - acc: 0.8704 - val_loss: 0.3697 - val_acc: 0.8398\n",
            "Epoch 124/500\n",
            "20108/20108 [==============================] - 13s 624us/sample - loss: 0.3105 - acc: 0.8704 - val_loss: 0.3689 - val_acc: 0.8418\n",
            "Epoch 125/500\n",
            "20108/20108 [==============================] - 13s 629us/sample - loss: 0.3125 - acc: 0.8698 - val_loss: 0.3692 - val_acc: 0.8416\n",
            "Epoch 126/500\n",
            "20108/20108 [==============================] - 13s 653us/sample - loss: 0.3091 - acc: 0.8723 - val_loss: 0.3687 - val_acc: 0.8420\n",
            "Epoch 127/500\n",
            "20108/20108 [==============================] - 13s 625us/sample - loss: 0.3094 - acc: 0.8701 - val_loss: 0.3688 - val_acc: 0.8420\n",
            "Epoch 128/500\n",
            "20108/20108 [==============================] - 13s 641us/sample - loss: 0.3079 - acc: 0.8730 - val_loss: 0.3686 - val_acc: 0.8426\n",
            "Epoch 129/500\n",
            "20108/20108 [==============================] - 13s 648us/sample - loss: 0.3058 - acc: 0.8716 - val_loss: 0.3691 - val_acc: 0.8416\n",
            "Epoch 130/500\n",
            "20108/20108 [==============================] - 13s 625us/sample - loss: 0.3042 - acc: 0.8715 - val_loss: 0.3694 - val_acc: 0.8414\n",
            "Epoch 131/500\n",
            "20108/20108 [==============================] - 13s 637us/sample - loss: 0.3032 - acc: 0.8760 - val_loss: 0.3692 - val_acc: 0.8420\n",
            "Epoch 132/500\n",
            "20108/20108 [==============================] - 13s 630us/sample - loss: 0.3059 - acc: 0.8752 - val_loss: 0.3680 - val_acc: 0.8428\n",
            "Epoch 133/500\n",
            "20108/20108 [==============================] - 12s 618us/sample - loss: 0.3051 - acc: 0.8744 - val_loss: 0.3683 - val_acc: 0.8428\n",
            "Epoch 134/500\n",
            "20108/20108 [==============================] - 13s 633us/sample - loss: 0.3011 - acc: 0.8748 - val_loss: 0.3687 - val_acc: 0.8432\n",
            "Epoch 135/500\n",
            "20108/20108 [==============================] - 12s 621us/sample - loss: 0.3018 - acc: 0.8738 - val_loss: 0.3683 - val_acc: 0.8432\n",
            "Epoch 136/500\n",
            "20108/20108 [==============================] - 13s 637us/sample - loss: 0.3018 - acc: 0.8767 - val_loss: 0.3691 - val_acc: 0.8428\n",
            "Epoch 137/500\n",
            "20108/20108 [==============================] - 13s 623us/sample - loss: 0.3000 - acc: 0.8749 - val_loss: 0.3694 - val_acc: 0.8432\n",
            "Epoch 138/500\n",
            "20108/20108 [==============================] - 13s 632us/sample - loss: 0.2981 - acc: 0.8763 - val_loss: 0.3685 - val_acc: 0.8436\n",
            "Epoch 139/500\n",
            "20108/20108 [==============================] - 13s 634us/sample - loss: 0.2966 - acc: 0.8779 - val_loss: 0.3700 - val_acc: 0.8426\n",
            "Epoch 00139: early stopping\n",
            "--- 1837.397940158844 seconds ---\n",
            "INFO:tensorflow:Copying TPU weights to the CPU\n",
            "20108/20108 [==============================] - 152s 8ms/sample - loss: 0.2788 - acc: 0.8850\n",
            "5028/5028 [==============================] - 38s 8ms/sample - loss: 0.3697 - acc: 0.8425\n",
            "Train Accuracy: 0.8850\n",
            "Train Loss: 0.2788\n",
            "Validation Accuracy:  0.8425\n",
            "Validation Loss: 0.3697\n",
            "5028/5028 [==============================] - 14s 3ms/sample\n",
            "[[ 991  509]\n",
            " [ 283 3245]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.78      0.66      0.71      1500\n",
            "         1.0       0.86      0.92      0.89      3528\n",
            "\n",
            "    accuracy                           0.84      5028\n",
            "   macro avg       0.82      0.79      0.80      5028\n",
            "weighted avg       0.84      0.84      0.84      5028\n",
            "\n",
            "RMSE: 0.3969\n",
            "--- Fold 2 ---\n",
            "INFO:tensorflow:Querying Tensorflow master (grpc://10.126.17.194:8470) for TPU system metadata.\n",
            "INFO:tensorflow:Found TPU system:\n",
            "INFO:tensorflow:*** Num TPU Cores: 8\n",
            "INFO:tensorflow:*** Num TPU Workers: 1\n",
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 12028669822908458194)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, 8434022613342494329)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 9483302766372785140)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, 16913650255987291235)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, 9584223271632520063)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 5131928026390797549)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 3800337932879369162)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 2724390565822717472)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 1277980390101900352)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, 15973569101757891556)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 8589934592, 13938748725989702413)\n",
            "WARNING:tensorflow:tpu_model (from tensorflow.contrib.tpu.python.tpu.keras_support) is experimental and may change or be removed at any time, and without warning.\n",
            "Train on 20109 samples, validate on 5027 samples\n",
            "Epoch 1/500\n",
            "INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(128, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(128, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 8.557979583740234 secs\n",
            "INFO:tensorflow:Setting weights on TPU model.\n",
            "18432/20109 [==========================>...] - ETA: 2s - loss: 0.6925 - acc: 0.5948INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(81,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(81, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(81, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 10.084686756134033 secs\n",
            "19456/20109 [============================>.] - ETA: 1s - loss: 0.6924 - acc: 0.6008INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(128, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(128, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 8.981791734695435 secs\n",
            "INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(116,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(116, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(116, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 10.914297103881836 secs\n",
            "20109/20109 [==============================] - 85s 4ms/sample - loss: 0.6923 - acc: 0.6040 - val_loss: 0.6910 - val_acc: 0.7020\n",
            "Epoch 2/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.6897 - acc: 0.7001 - val_loss: 0.6881 - val_acc: 0.7020\n",
            "Epoch 3/500\n",
            "20109/20109 [==============================] - 12s 621us/sample - loss: 0.6864 - acc: 0.7016 - val_loss: 0.6843 - val_acc: 0.7020\n",
            "Epoch 4/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.6820 - acc: 0.7017 - val_loss: 0.6793 - val_acc: 0.7020\n",
            "Epoch 5/500\n",
            "20109/20109 [==============================] - 13s 645us/sample - loss: 0.6762 - acc: 0.7017 - val_loss: 0.6726 - val_acc: 0.7020\n",
            "Epoch 6/500\n",
            "20109/20109 [==============================] - 13s 624us/sample - loss: 0.6685 - acc: 0.7016 - val_loss: 0.6636 - val_acc: 0.7020\n",
            "Epoch 7/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.6579 - acc: 0.7016 - val_loss: 0.6518 - val_acc: 0.7020\n",
            "Epoch 8/500\n",
            "20109/20109 [==============================] - 12s 620us/sample - loss: 0.6452 - acc: 0.7016 - val_loss: 0.6377 - val_acc: 0.7020\n",
            "Epoch 9/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.6320 - acc: 0.7016 - val_loss: 0.6250 - val_acc: 0.7020\n",
            "Epoch 10/500\n",
            "20109/20109 [==============================] - 13s 623us/sample - loss: 0.6215 - acc: 0.7016 - val_loss: 0.6181 - val_acc: 0.7020\n",
            "Epoch 11/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.6182 - acc: 0.7016 - val_loss: 0.6147 - val_acc: 0.7020\n",
            "Epoch 12/500\n",
            "20109/20109 [==============================] - 12s 622us/sample - loss: 0.6149 - acc: 0.7017 - val_loss: 0.6120 - val_acc: 0.7020\n",
            "Epoch 13/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.6129 - acc: 0.7017 - val_loss: 0.6098 - val_acc: 0.7020\n",
            "Epoch 14/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.6097 - acc: 0.7016 - val_loss: 0.6079 - val_acc: 0.7020\n",
            "Epoch 15/500\n",
            "20109/20109 [==============================] - 13s 624us/sample - loss: 0.6090 - acc: 0.7016 - val_loss: 0.6062 - val_acc: 0.7020\n",
            "Epoch 16/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.6070 - acc: 0.7017 - val_loss: 0.6047 - val_acc: 0.7020\n",
            "Epoch 17/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.6051 - acc: 0.7016 - val_loss: 0.6032 - val_acc: 0.7020\n",
            "Epoch 18/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.6046 - acc: 0.7017 - val_loss: 0.6017 - val_acc: 0.7020\n",
            "Epoch 19/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.6026 - acc: 0.7016 - val_loss: 0.6000 - val_acc: 0.7020\n",
            "Epoch 20/500\n",
            "20109/20109 [==============================] - 12s 621us/sample - loss: 0.6004 - acc: 0.7017 - val_loss: 0.5981 - val_acc: 0.7020\n",
            "Epoch 21/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.5982 - acc: 0.7017 - val_loss: 0.5960 - val_acc: 0.7020\n",
            "Epoch 22/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.5959 - acc: 0.7017 - val_loss: 0.5937 - val_acc: 0.7020\n",
            "Epoch 23/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.5931 - acc: 0.7016 - val_loss: 0.5909 - val_acc: 0.7020\n",
            "Epoch 24/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.5911 - acc: 0.7017 - val_loss: 0.5877 - val_acc: 0.7020\n",
            "Epoch 25/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.5870 - acc: 0.7017 - val_loss: 0.5837 - val_acc: 0.7020\n",
            "Epoch 26/500\n",
            "20109/20109 [==============================] - 13s 650us/sample - loss: 0.5834 - acc: 0.7016 - val_loss: 0.5788 - val_acc: 0.7020\n",
            "Epoch 27/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.5778 - acc: 0.7017 - val_loss: 0.5732 - val_acc: 0.7020\n",
            "Epoch 28/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.5734 - acc: 0.7018 - val_loss: 0.5669 - val_acc: 0.7020\n",
            "Epoch 29/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.5650 - acc: 0.7019 - val_loss: 0.5592 - val_acc: 0.7020\n",
            "Epoch 30/500\n",
            "20109/20109 [==============================] - 12s 621us/sample - loss: 0.5581 - acc: 0.7025 - val_loss: 0.5507 - val_acc: 0.7024\n",
            "Epoch 31/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.5514 - acc: 0.7037 - val_loss: 0.5416 - val_acc: 0.7038\n",
            "Epoch 32/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.5409 - acc: 0.7096 - val_loss: 0.5300 - val_acc: 0.7068\n",
            "Epoch 33/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.5323 - acc: 0.7170 - val_loss: 0.5190 - val_acc: 0.7164\n",
            "Epoch 34/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.5218 - acc: 0.7269 - val_loss: 0.5074 - val_acc: 0.7297\n",
            "Epoch 35/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.5126 - acc: 0.7400 - val_loss: 0.4973 - val_acc: 0.7430\n",
            "Epoch 36/500\n",
            "20109/20109 [==============================] - 12s 614us/sample - loss: 0.5043 - acc: 0.7469 - val_loss: 0.4869 - val_acc: 0.7530\n",
            "Epoch 37/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.4924 - acc: 0.7609 - val_loss: 0.4771 - val_acc: 0.7600\n",
            "Epoch 38/500\n",
            "20109/20109 [==============================] - 12s 619us/sample - loss: 0.4879 - acc: 0.7653 - val_loss: 0.4714 - val_acc: 0.7681\n",
            "Epoch 39/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.4809 - acc: 0.7732 - val_loss: 0.4652 - val_acc: 0.7703\n",
            "Epoch 40/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.4741 - acc: 0.7742 - val_loss: 0.4601 - val_acc: 0.7771\n",
            "Epoch 41/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.4665 - acc: 0.7816 - val_loss: 0.4551 - val_acc: 0.7777\n",
            "Epoch 42/500\n",
            "20109/20109 [==============================] - 13s 622us/sample - loss: 0.4652 - acc: 0.7845 - val_loss: 0.4518 - val_acc: 0.7783\n",
            "Epoch 43/500\n",
            "20109/20109 [==============================] - 13s 623us/sample - loss: 0.4606 - acc: 0.7874 - val_loss: 0.4491 - val_acc: 0.7860\n",
            "Epoch 44/500\n",
            "20109/20109 [==============================] - 13s 645us/sample - loss: 0.4529 - acc: 0.7905 - val_loss: 0.4443 - val_acc: 0.7838\n",
            "Epoch 45/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.4481 - acc: 0.7915 - val_loss: 0.4410 - val_acc: 0.7880\n",
            "Epoch 46/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.4473 - acc: 0.7905 - val_loss: 0.4391 - val_acc: 0.7876\n",
            "Epoch 47/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.4387 - acc: 0.7979 - val_loss: 0.4351 - val_acc: 0.7902\n",
            "Epoch 48/500\n",
            "20109/20109 [==============================] - 13s 653us/sample - loss: 0.4379 - acc: 0.7980 - val_loss: 0.4331 - val_acc: 0.7908\n",
            "Epoch 49/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.4328 - acc: 0.8016 - val_loss: 0.4309 - val_acc: 0.7910\n",
            "Epoch 50/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.4271 - acc: 0.8026 - val_loss: 0.4290 - val_acc: 0.7910\n",
            "Epoch 51/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.4239 - acc: 0.8048 - val_loss: 0.4265 - val_acc: 0.7934\n",
            "Epoch 52/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.4200 - acc: 0.8075 - val_loss: 0.4245 - val_acc: 0.7964\n",
            "Epoch 53/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.4157 - acc: 0.8116 - val_loss: 0.4223 - val_acc: 0.7998\n",
            "Epoch 54/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.4162 - acc: 0.8086 - val_loss: 0.4216 - val_acc: 0.7998\n",
            "Epoch 55/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.4144 - acc: 0.8093 - val_loss: 0.4199 - val_acc: 0.8008\n",
            "Epoch 56/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.4076 - acc: 0.8161 - val_loss: 0.4185 - val_acc: 0.7994\n",
            "Epoch 57/500\n",
            "20109/20109 [==============================] - 13s 648us/sample - loss: 0.4043 - acc: 0.8168 - val_loss: 0.4173 - val_acc: 0.7992\n",
            "Epoch 58/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.4020 - acc: 0.8167 - val_loss: 0.4151 - val_acc: 0.8035\n",
            "Epoch 59/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.3996 - acc: 0.8188 - val_loss: 0.4137 - val_acc: 0.8043\n",
            "Epoch 60/500\n",
            "20109/20109 [==============================] - 13s 623us/sample - loss: 0.3989 - acc: 0.8213 - val_loss: 0.4124 - val_acc: 0.8047\n",
            "Epoch 61/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.3971 - acc: 0.8225 - val_loss: 0.4114 - val_acc: 0.8059\n",
            "Epoch 62/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.3916 - acc: 0.8262 - val_loss: 0.4096 - val_acc: 0.8101\n",
            "Epoch 63/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.3896 - acc: 0.8244 - val_loss: 0.4090 - val_acc: 0.8071\n",
            "Epoch 64/500\n",
            "20109/20109 [==============================] - 13s 624us/sample - loss: 0.3867 - acc: 0.8262 - val_loss: 0.4076 - val_acc: 0.8087\n",
            "Epoch 65/500\n",
            "20109/20109 [==============================] - 13s 647us/sample - loss: 0.3878 - acc: 0.8265 - val_loss: 0.4065 - val_acc: 0.8099\n",
            "Epoch 66/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.3862 - acc: 0.8276 - val_loss: 0.4054 - val_acc: 0.8103\n",
            "Epoch 67/500\n",
            "20109/20109 [==============================] - 12s 618us/sample - loss: 0.3817 - acc: 0.8288 - val_loss: 0.4042 - val_acc: 0.8113\n",
            "Epoch 68/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.3817 - acc: 0.8296 - val_loss: 0.4039 - val_acc: 0.8107\n",
            "Epoch 69/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3765 - acc: 0.8334 - val_loss: 0.4016 - val_acc: 0.8151\n",
            "Epoch 70/500\n",
            "20109/20109 [==============================] - 12s 621us/sample - loss: 0.3732 - acc: 0.8361 - val_loss: 0.4009 - val_acc: 0.8123\n",
            "Epoch 71/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.3757 - acc: 0.8333 - val_loss: 0.4002 - val_acc: 0.8127\n",
            "Epoch 72/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.3712 - acc: 0.8356 - val_loss: 0.3986 - val_acc: 0.8159\n",
            "Epoch 73/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.3727 - acc: 0.8355 - val_loss: 0.3984 - val_acc: 0.8135\n",
            "Epoch 74/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3664 - acc: 0.8381 - val_loss: 0.3975 - val_acc: 0.8157\n",
            "Epoch 75/500\n",
            "20109/20109 [==============================] - 12s 613us/sample - loss: 0.3663 - acc: 0.8381 - val_loss: 0.3963 - val_acc: 0.8169\n",
            "Epoch 76/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.3647 - acc: 0.8378 - val_loss: 0.3954 - val_acc: 0.8173\n",
            "Epoch 77/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.3619 - acc: 0.8406 - val_loss: 0.3944 - val_acc: 0.8187\n",
            "Epoch 78/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.3592 - acc: 0.8419 - val_loss: 0.3937 - val_acc: 0.8199\n",
            "Epoch 79/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.3588 - acc: 0.8431 - val_loss: 0.3929 - val_acc: 0.8213\n",
            "Epoch 80/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.3575 - acc: 0.8440 - val_loss: 0.3921 - val_acc: 0.8223\n",
            "Epoch 81/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.3574 - acc: 0.8415 - val_loss: 0.3922 - val_acc: 0.8213\n",
            "Epoch 82/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.3544 - acc: 0.8465 - val_loss: 0.3906 - val_acc: 0.8230\n",
            "Epoch 83/500\n",
            "20109/20109 [==============================] - 12s 618us/sample - loss: 0.3492 - acc: 0.8488 - val_loss: 0.3905 - val_acc: 0.8236\n",
            "Epoch 84/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.3510 - acc: 0.8462 - val_loss: 0.3890 - val_acc: 0.8238\n",
            "Epoch 85/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.3517 - acc: 0.8459 - val_loss: 0.3882 - val_acc: 0.8248\n",
            "Epoch 86/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3459 - acc: 0.8497 - val_loss: 0.3895 - val_acc: 0.8234\n",
            "Epoch 87/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.3484 - acc: 0.8471 - val_loss: 0.3869 - val_acc: 0.8254\n",
            "Epoch 88/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3417 - acc: 0.8534 - val_loss: 0.3862 - val_acc: 0.8268\n",
            "Epoch 89/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.3415 - acc: 0.8530 - val_loss: 0.3858 - val_acc: 0.8254\n",
            "Epoch 90/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.3450 - acc: 0.8497 - val_loss: 0.3847 - val_acc: 0.8270\n",
            "Epoch 91/500\n",
            "20109/20109 [==============================] - 12s 619us/sample - loss: 0.3420 - acc: 0.8544 - val_loss: 0.3842 - val_acc: 0.8274\n",
            "Epoch 92/500\n",
            "20109/20109 [==============================] - 12s 615us/sample - loss: 0.3386 - acc: 0.8545 - val_loss: 0.3840 - val_acc: 0.8282\n",
            "Epoch 93/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3408 - acc: 0.8530 - val_loss: 0.3833 - val_acc: 0.8286\n",
            "Epoch 94/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.3373 - acc: 0.8558 - val_loss: 0.3830 - val_acc: 0.8286\n",
            "Epoch 95/500\n",
            "20109/20109 [==============================] - 13s 622us/sample - loss: 0.3358 - acc: 0.8567 - val_loss: 0.3832 - val_acc: 0.8278\n",
            "Epoch 96/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3349 - acc: 0.8560 - val_loss: 0.3819 - val_acc: 0.8290\n",
            "Epoch 97/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.3335 - acc: 0.8584 - val_loss: 0.3816 - val_acc: 0.8294\n",
            "Epoch 98/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3311 - acc: 0.8602 - val_loss: 0.3810 - val_acc: 0.8290\n",
            "Epoch 99/500\n",
            "20109/20109 [==============================] - 13s 644us/sample - loss: 0.3317 - acc: 0.8582 - val_loss: 0.3802 - val_acc: 0.8304\n",
            "Epoch 100/500\n",
            "20109/20109 [==============================] - 12s 620us/sample - loss: 0.3288 - acc: 0.8587 - val_loss: 0.3797 - val_acc: 0.8302\n",
            "Epoch 101/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.3304 - acc: 0.8602 - val_loss: 0.3796 - val_acc: 0.8294\n",
            "Epoch 102/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.3280 - acc: 0.8614 - val_loss: 0.3790 - val_acc: 0.8304\n",
            "Epoch 103/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.3256 - acc: 0.8608 - val_loss: 0.3787 - val_acc: 0.8310\n",
            "Epoch 104/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.3272 - acc: 0.8614 - val_loss: 0.3785 - val_acc: 0.8308\n",
            "Epoch 105/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3255 - acc: 0.8628 - val_loss: 0.3779 - val_acc: 0.8318\n",
            "Epoch 106/500\n",
            "20109/20109 [==============================] - 13s 644us/sample - loss: 0.3228 - acc: 0.8654 - val_loss: 0.3773 - val_acc: 0.8318\n",
            "Epoch 107/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3238 - acc: 0.8632 - val_loss: 0.3767 - val_acc: 0.8326\n",
            "Epoch 108/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.3218 - acc: 0.8660 - val_loss: 0.3763 - val_acc: 0.8332\n",
            "Epoch 109/500\n",
            "20109/20109 [==============================] - 12s 616us/sample - loss: 0.3225 - acc: 0.8625 - val_loss: 0.3767 - val_acc: 0.8310\n",
            "Epoch 110/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.3206 - acc: 0.8648 - val_loss: 0.3758 - val_acc: 0.8336\n",
            "Epoch 111/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.3206 - acc: 0.8633 - val_loss: 0.3763 - val_acc: 0.8310\n",
            "Epoch 112/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.3156 - acc: 0.8675 - val_loss: 0.3758 - val_acc: 0.8332\n",
            "Epoch 113/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.3185 - acc: 0.8654 - val_loss: 0.3760 - val_acc: 0.8314\n",
            "Epoch 114/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3216 - acc: 0.8642 - val_loss: 0.3751 - val_acc: 0.8338\n",
            "Epoch 115/500\n",
            "20109/20109 [==============================] - 13s 624us/sample - loss: 0.3166 - acc: 0.8654 - val_loss: 0.3747 - val_acc: 0.8342\n",
            "Epoch 116/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.3152 - acc: 0.8667 - val_loss: 0.3746 - val_acc: 0.8348\n",
            "Epoch 117/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.3159 - acc: 0.8661 - val_loss: 0.3747 - val_acc: 0.8356\n",
            "Epoch 118/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.3117 - acc: 0.8674 - val_loss: 0.3741 - val_acc: 0.8344\n",
            "Epoch 119/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.3128 - acc: 0.8690 - val_loss: 0.3745 - val_acc: 0.8364\n",
            "Epoch 120/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.3117 - acc: 0.8689 - val_loss: 0.3739 - val_acc: 0.8348\n",
            "Epoch 121/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.3098 - acc: 0.8675 - val_loss: 0.3735 - val_acc: 0.8352\n",
            "Epoch 122/500\n",
            "20109/20109 [==============================] - 12s 615us/sample - loss: 0.3115 - acc: 0.8694 - val_loss: 0.3733 - val_acc: 0.8358\n",
            "Epoch 123/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.3085 - acc: 0.8711 - val_loss: 0.3735 - val_acc: 0.8358\n",
            "Epoch 124/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.3081 - acc: 0.8715 - val_loss: 0.3729 - val_acc: 0.8368\n",
            "Epoch 125/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.3083 - acc: 0.8718 - val_loss: 0.3726 - val_acc: 0.8364\n",
            "Epoch 126/500\n",
            "20109/20109 [==============================] - 12s 619us/sample - loss: 0.3074 - acc: 0.8714 - val_loss: 0.3725 - val_acc: 0.8368\n",
            "Epoch 127/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.3038 - acc: 0.8730 - val_loss: 0.3726 - val_acc: 0.8364\n",
            "Epoch 128/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.3033 - acc: 0.8720 - val_loss: 0.3730 - val_acc: 0.8366\n",
            "Epoch 129/500\n",
            "20109/20109 [==============================] - 13s 645us/sample - loss: 0.3048 - acc: 0.8706 - val_loss: 0.3725 - val_acc: 0.8374\n",
            "Epoch 130/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.3020 - acc: 0.8723 - val_loss: 0.3733 - val_acc: 0.8366\n",
            "Epoch 131/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3044 - acc: 0.8735 - val_loss: 0.3725 - val_acc: 0.8364\n",
            "Epoch 132/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.3022 - acc: 0.8755 - val_loss: 0.3725 - val_acc: 0.8386\n",
            "Epoch 133/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.3013 - acc: 0.8732 - val_loss: 0.3717 - val_acc: 0.8378\n",
            "Epoch 134/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.3028 - acc: 0.8731 - val_loss: 0.3715 - val_acc: 0.8376\n",
            "Epoch 135/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3011 - acc: 0.8760 - val_loss: 0.3715 - val_acc: 0.8394\n",
            "Epoch 136/500\n",
            "20109/20109 [==============================] - 12s 621us/sample - loss: 0.2971 - acc: 0.8769 - val_loss: 0.3715 - val_acc: 0.8390\n",
            "Epoch 137/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.2977 - acc: 0.8766 - val_loss: 0.3711 - val_acc: 0.8390\n",
            "Epoch 138/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.2980 - acc: 0.8763 - val_loss: 0.3715 - val_acc: 0.8378\n",
            "Epoch 139/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.2980 - acc: 0.8746 - val_loss: 0.3705 - val_acc: 0.8380\n",
            "Epoch 140/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.2953 - acc: 0.8760 - val_loss: 0.3710 - val_acc: 0.8386\n",
            "Epoch 141/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.2981 - acc: 0.8777 - val_loss: 0.3710 - val_acc: 0.8384\n",
            "Epoch 142/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.2953 - acc: 0.8793 - val_loss: 0.3708 - val_acc: 0.8388\n",
            "Epoch 143/500\n",
            "20109/20109 [==============================] - 13s 647us/sample - loss: 0.2936 - acc: 0.8790 - val_loss: 0.3717 - val_acc: 0.8386\n",
            "Epoch 144/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.2954 - acc: 0.8765 - val_loss: 0.3706 - val_acc: 0.8392\n",
            "Epoch 145/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.2947 - acc: 0.8786 - val_loss: 0.3704 - val_acc: 0.8388\n",
            "Epoch 146/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.2938 - acc: 0.8795 - val_loss: 0.3702 - val_acc: 0.8390\n",
            "Epoch 147/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.2925 - acc: 0.8777 - val_loss: 0.3705 - val_acc: 0.8386\n",
            "Epoch 148/500\n",
            "20109/20109 [==============================] - 13s 623us/sample - loss: 0.2885 - acc: 0.8810 - val_loss: 0.3704 - val_acc: 0.8390\n",
            "Epoch 149/500\n",
            "20109/20109 [==============================] - 13s 644us/sample - loss: 0.2902 - acc: 0.8802 - val_loss: 0.3702 - val_acc: 0.8392\n",
            "Epoch 150/500\n",
            "20109/20109 [==============================] - 12s 613us/sample - loss: 0.2916 - acc: 0.8811 - val_loss: 0.3706 - val_acc: 0.8382\n",
            "Epoch 151/500\n",
            "20109/20109 [==============================] - 13s 624us/sample - loss: 0.2901 - acc: 0.8796 - val_loss: 0.3708 - val_acc: 0.8388\n",
            "Epoch 152/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.2902 - acc: 0.8812 - val_loss: 0.3699 - val_acc: 0.8394\n",
            "Epoch 153/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.2894 - acc: 0.8808 - val_loss: 0.3700 - val_acc: 0.8388\n",
            "Epoch 154/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.2879 - acc: 0.8803 - val_loss: 0.3702 - val_acc: 0.8400\n",
            "Epoch 155/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.2863 - acc: 0.8823 - val_loss: 0.3702 - val_acc: 0.8394\n",
            "Epoch 156/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.2868 - acc: 0.8828 - val_loss: 0.3704 - val_acc: 0.8382\n",
            "Epoch 157/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.2874 - acc: 0.8827 - val_loss: 0.3703 - val_acc: 0.8386\n",
            "Epoch 158/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.2849 - acc: 0.8819 - val_loss: 0.3700 - val_acc: 0.8382\n",
            "Epoch 159/500\n",
            "20109/20109 [==============================] - 13s 623us/sample - loss: 0.2844 - acc: 0.8838 - val_loss: 0.3706 - val_acc: 0.8384\n",
            "Epoch 00159: early stopping\n",
            "--- 2093.989465236664 seconds ---\n",
            "INFO:tensorflow:Copying TPU weights to the CPU\n",
            "20109/20109 [==============================] - 153s 8ms/sample - loss: 0.2658 - acc: 0.8906\n",
            "5027/5027 [==============================] - 38s 8ms/sample - loss: 0.3704 - acc: 0.8385\n",
            "Train Accuracy: 0.8906\n",
            "Train Loss: 0.2658\n",
            "Validation Accuracy:  0.8385\n",
            "Validation Loss: 0.3704\n",
            "5027/5027 [==============================] - 14s 3ms/sample\n",
            "[[1012  487]\n",
            " [ 325 3203]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.76      0.68      0.71      1499\n",
            "         1.0       0.87      0.91      0.89      3528\n",
            "\n",
            "    accuracy                           0.84      5027\n",
            "   macro avg       0.81      0.79      0.80      5027\n",
            "weighted avg       0.83      0.84      0.84      5027\n",
            "\n",
            "RMSE: 0.4019\n",
            "--- Fold 3 ---\n",
            "INFO:tensorflow:Querying Tensorflow master (grpc://10.126.17.194:8470) for TPU system metadata.\n",
            "INFO:tensorflow:Found TPU system:\n",
            "INFO:tensorflow:*** Num TPU Cores: 8\n",
            "INFO:tensorflow:*** Num TPU Workers: 1\n",
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 12028669822908458194)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, 8434022613342494329)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 9483302766372785140)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, 16913650255987291235)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, 9584223271632520063)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 5131928026390797549)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 3800337932879369162)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 2724390565822717472)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 1277980390101900352)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, 15973569101757891556)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 8589934592, 13938748725989702413)\n",
            "WARNING:tensorflow:tpu_model (from tensorflow.contrib.tpu.python.tpu.keras_support) is experimental and may change or be removed at any time, and without warning.\n",
            "Train on 20109 samples, validate on 5027 samples\n",
            "Epoch 1/500\n",
            "INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(128, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(128, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 9.222805500030518 secs\n",
            "INFO:tensorflow:Setting weights on TPU model.\n",
            "18432/20109 [==========================>...] - ETA: 3s - loss: 0.6923 - acc: 0.6105INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(81,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(81, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(81, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 10.967965841293335 secs\n",
            "19456/20109 [============================>.] - ETA: 1s - loss: 0.6922 - acc: 0.6153INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(128, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(128, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 10.018864870071411 secs\n",
            "INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(116,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(116, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(116, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 12.448522567749023 secs\n",
            "20109/20109 [==============================] - 90s 4ms/sample - loss: 0.6922 - acc: 0.6184 - val_loss: 0.6908 - val_acc: 0.7016\n",
            "Epoch 2/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.6894 - acc: 0.7010 - val_loss: 0.6876 - val_acc: 0.7016\n",
            "Epoch 3/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.6858 - acc: 0.7016 - val_loss: 0.6836 - val_acc: 0.7016\n",
            "Epoch 4/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.6812 - acc: 0.7017 - val_loss: 0.6781 - val_acc: 0.7016\n",
            "Epoch 5/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.6748 - acc: 0.7017 - val_loss: 0.6707 - val_acc: 0.7016\n",
            "Epoch 6/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.6663 - acc: 0.7017 - val_loss: 0.6607 - val_acc: 0.7016\n",
            "Epoch 7/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.6554 - acc: 0.7018 - val_loss: 0.6478 - val_acc: 0.7016\n",
            "Epoch 8/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.6413 - acc: 0.7018 - val_loss: 0.6333 - val_acc: 0.7016\n",
            "Epoch 9/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.6294 - acc: 0.7017 - val_loss: 0.6231 - val_acc: 0.7016\n",
            "Epoch 10/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.6223 - acc: 0.7017 - val_loss: 0.6182 - val_acc: 0.7016\n",
            "Epoch 11/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.6187 - acc: 0.7017 - val_loss: 0.6150 - val_acc: 0.7016\n",
            "Epoch 12/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.6162 - acc: 0.7016 - val_loss: 0.6124 - val_acc: 0.7016\n",
            "Epoch 13/500\n",
            "20109/20109 [==============================] - 13s 623us/sample - loss: 0.6128 - acc: 0.7018 - val_loss: 0.6103 - val_acc: 0.7016\n",
            "Epoch 14/500\n",
            "20109/20109 [==============================] - 12s 619us/sample - loss: 0.6115 - acc: 0.7017 - val_loss: 0.6086 - val_acc: 0.7016\n",
            "Epoch 15/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.6098 - acc: 0.7017 - val_loss: 0.6071 - val_acc: 0.7016\n",
            "Epoch 16/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.6084 - acc: 0.7017 - val_loss: 0.6057 - val_acc: 0.7016\n",
            "Epoch 17/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.6064 - acc: 0.7017 - val_loss: 0.6044 - val_acc: 0.7016\n",
            "Epoch 18/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.6057 - acc: 0.7017 - val_loss: 0.6032 - val_acc: 0.7016\n",
            "Epoch 19/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.6048 - acc: 0.7017 - val_loss: 0.6017 - val_acc: 0.7016\n",
            "Epoch 20/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.6028 - acc: 0.7017 - val_loss: 0.6004 - val_acc: 0.7016\n",
            "Epoch 21/500\n",
            "20109/20109 [==============================] - 12s 617us/sample - loss: 0.6017 - acc: 0.7017 - val_loss: 0.5986 - val_acc: 0.7016\n",
            "Epoch 22/500\n",
            "20109/20109 [==============================] - 13s 644us/sample - loss: 0.5997 - acc: 0.7016 - val_loss: 0.5967 - val_acc: 0.7016\n",
            "Epoch 23/500\n",
            "20109/20109 [==============================] - 13s 622us/sample - loss: 0.5970 - acc: 0.7017 - val_loss: 0.5944 - val_acc: 0.7016\n",
            "Epoch 24/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.5958 - acc: 0.7016 - val_loss: 0.5922 - val_acc: 0.7016\n",
            "Epoch 25/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.5917 - acc: 0.7017 - val_loss: 0.5890 - val_acc: 0.7016\n",
            "Epoch 26/500\n",
            "20109/20109 [==============================] - 13s 647us/sample - loss: 0.5883 - acc: 0.7016 - val_loss: 0.5851 - val_acc: 0.7016\n",
            "Epoch 27/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.5846 - acc: 0.7017 - val_loss: 0.5808 - val_acc: 0.7016\n",
            "Epoch 28/500\n",
            "20109/20109 [==============================] - 13s 648us/sample - loss: 0.5797 - acc: 0.7017 - val_loss: 0.5755 - val_acc: 0.7016\n",
            "Epoch 29/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.5748 - acc: 0.7018 - val_loss: 0.5699 - val_acc: 0.7016\n",
            "Epoch 30/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.5693 - acc: 0.7019 - val_loss: 0.5633 - val_acc: 0.7016\n",
            "Epoch 31/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.5625 - acc: 0.7028 - val_loss: 0.5546 - val_acc: 0.7016\n",
            "Epoch 32/500\n",
            "20109/20109 [==============================] - 12s 620us/sample - loss: 0.5532 - acc: 0.7035 - val_loss: 0.5438 - val_acc: 0.7014\n",
            "Epoch 33/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.5449 - acc: 0.7068 - val_loss: 0.5318 - val_acc: 0.7054\n",
            "Epoch 34/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.5341 - acc: 0.7125 - val_loss: 0.5203 - val_acc: 0.7156\n",
            "Epoch 35/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.5248 - acc: 0.7227 - val_loss: 0.5083 - val_acc: 0.7283\n",
            "Epoch 36/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.5126 - acc: 0.7371 - val_loss: 0.4955 - val_acc: 0.7440\n",
            "Epoch 37/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.5072 - acc: 0.7478 - val_loss: 0.4862 - val_acc: 0.7578\n",
            "Epoch 38/500\n",
            "20109/20109 [==============================] - 12s 613us/sample - loss: 0.4958 - acc: 0.7586 - val_loss: 0.4782 - val_acc: 0.7695\n",
            "Epoch 39/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.4934 - acc: 0.7644 - val_loss: 0.4711 - val_acc: 0.7739\n",
            "Epoch 40/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.4863 - acc: 0.7694 - val_loss: 0.4653 - val_acc: 0.7805\n",
            "Epoch 41/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.4769 - acc: 0.7756 - val_loss: 0.4601 - val_acc: 0.7832\n",
            "Epoch 42/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.4734 - acc: 0.7756 - val_loss: 0.4558 - val_acc: 0.7836\n",
            "Epoch 43/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.4654 - acc: 0.7789 - val_loss: 0.4515 - val_acc: 0.7870\n",
            "Epoch 44/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.4594 - acc: 0.7883 - val_loss: 0.4473 - val_acc: 0.7848\n",
            "Epoch 45/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.4563 - acc: 0.7872 - val_loss: 0.4438 - val_acc: 0.7874\n",
            "Epoch 46/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.4496 - acc: 0.7938 - val_loss: 0.4408 - val_acc: 0.7888\n",
            "Epoch 47/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.4459 - acc: 0.7930 - val_loss: 0.4385 - val_acc: 0.7870\n",
            "Epoch 48/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.4381 - acc: 0.7967 - val_loss: 0.4362 - val_acc: 0.7948\n",
            "Epoch 49/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.4374 - acc: 0.7964 - val_loss: 0.4332 - val_acc: 0.7916\n",
            "Epoch 50/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.4343 - acc: 0.8014 - val_loss: 0.4307 - val_acc: 0.7938\n",
            "Epoch 51/500\n",
            "20109/20109 [==============================] - 13s 645us/sample - loss: 0.4281 - acc: 0.8049 - val_loss: 0.4290 - val_acc: 0.7938\n",
            "Epoch 52/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.4239 - acc: 0.8055 - val_loss: 0.4263 - val_acc: 0.7998\n",
            "Epoch 53/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.4234 - acc: 0.8074 - val_loss: 0.4252 - val_acc: 0.7982\n",
            "Epoch 54/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.4179 - acc: 0.8050 - val_loss: 0.4241 - val_acc: 0.7962\n",
            "Epoch 55/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.4133 - acc: 0.8098 - val_loss: 0.4221 - val_acc: 0.8025\n",
            "Epoch 56/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.4141 - acc: 0.8146 - val_loss: 0.4203 - val_acc: 0.8031\n",
            "Epoch 57/500\n",
            "20109/20109 [==============================] - 13s 624us/sample - loss: 0.4104 - acc: 0.8131 - val_loss: 0.4190 - val_acc: 0.8047\n",
            "Epoch 58/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.4085 - acc: 0.8155 - val_loss: 0.4179 - val_acc: 0.8041\n",
            "Epoch 59/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.4057 - acc: 0.8186 - val_loss: 0.4170 - val_acc: 0.8025\n",
            "Epoch 60/500\n",
            "20109/20109 [==============================] - 12s 617us/sample - loss: 0.4044 - acc: 0.8179 - val_loss: 0.4156 - val_acc: 0.8055\n",
            "Epoch 61/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.3996 - acc: 0.8207 - val_loss: 0.4142 - val_acc: 0.8063\n",
            "Epoch 62/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.3965 - acc: 0.8204 - val_loss: 0.4131 - val_acc: 0.8067\n",
            "Epoch 63/500\n",
            "20109/20109 [==============================] - 12s 614us/sample - loss: 0.3940 - acc: 0.8211 - val_loss: 0.4118 - val_acc: 0.8083\n",
            "Epoch 64/500\n",
            "20109/20109 [==============================] - 12s 621us/sample - loss: 0.3900 - acc: 0.8269 - val_loss: 0.4104 - val_acc: 0.8091\n",
            "Epoch 65/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.3887 - acc: 0.8242 - val_loss: 0.4095 - val_acc: 0.8101\n",
            "Epoch 66/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.3866 - acc: 0.8284 - val_loss: 0.4091 - val_acc: 0.8095\n",
            "Epoch 67/500\n",
            "20109/20109 [==============================] - 13s 646us/sample - loss: 0.3821 - acc: 0.8291 - val_loss: 0.4074 - val_acc: 0.8121\n",
            "Epoch 68/500\n",
            "20109/20109 [==============================] - 13s 622us/sample - loss: 0.3818 - acc: 0.8295 - val_loss: 0.4065 - val_acc: 0.8139\n",
            "Epoch 69/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.3802 - acc: 0.8300 - val_loss: 0.4056 - val_acc: 0.8125\n",
            "Epoch 70/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.3757 - acc: 0.8323 - val_loss: 0.4043 - val_acc: 0.8139\n",
            "Epoch 71/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.3761 - acc: 0.8324 - val_loss: 0.4034 - val_acc: 0.8143\n",
            "Epoch 72/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.3739 - acc: 0.8354 - val_loss: 0.4033 - val_acc: 0.8141\n",
            "Epoch 73/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3719 - acc: 0.8335 - val_loss: 0.4020 - val_acc: 0.8161\n",
            "Epoch 74/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.3704 - acc: 0.8375 - val_loss: 0.4009 - val_acc: 0.8151\n",
            "Epoch 75/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.3667 - acc: 0.8372 - val_loss: 0.4003 - val_acc: 0.8169\n",
            "Epoch 76/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3664 - acc: 0.8378 - val_loss: 0.4000 - val_acc: 0.8161\n",
            "Epoch 77/500\n",
            "20109/20109 [==============================] - 13s 652us/sample - loss: 0.3650 - acc: 0.8392 - val_loss: 0.3995 - val_acc: 0.8171\n",
            "Epoch 78/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.3624 - acc: 0.8427 - val_loss: 0.3979 - val_acc: 0.8205\n",
            "Epoch 79/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.3626 - acc: 0.8404 - val_loss: 0.3974 - val_acc: 0.8181\n",
            "Epoch 80/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.3581 - acc: 0.8428 - val_loss: 0.3965 - val_acc: 0.8217\n",
            "Epoch 81/500\n",
            "20109/20109 [==============================] - 12s 620us/sample - loss: 0.3564 - acc: 0.8430 - val_loss: 0.3967 - val_acc: 0.8195\n",
            "Epoch 82/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3567 - acc: 0.8451 - val_loss: 0.3951 - val_acc: 0.8205\n",
            "Epoch 83/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.3554 - acc: 0.8431 - val_loss: 0.3949 - val_acc: 0.8215\n",
            "Epoch 84/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.3527 - acc: 0.8446 - val_loss: 0.3936 - val_acc: 0.8221\n",
            "Epoch 85/500\n",
            "20109/20109 [==============================] - 13s 646us/sample - loss: 0.3502 - acc: 0.8489 - val_loss: 0.3930 - val_acc: 0.8225\n",
            "Epoch 86/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3483 - acc: 0.8499 - val_loss: 0.3923 - val_acc: 0.8225\n",
            "Epoch 87/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.3448 - acc: 0.8499 - val_loss: 0.3917 - val_acc: 0.8229\n",
            "Epoch 88/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.3484 - acc: 0.8478 - val_loss: 0.3912 - val_acc: 0.8240\n",
            "Epoch 89/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.3454 - acc: 0.8503 - val_loss: 0.3917 - val_acc: 0.8252\n",
            "Epoch 90/500\n",
            "20109/20109 [==============================] - 13s 647us/sample - loss: 0.3458 - acc: 0.8500 - val_loss: 0.3903 - val_acc: 0.8250\n",
            "Epoch 91/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3417 - acc: 0.8502 - val_loss: 0.3898 - val_acc: 0.8254\n",
            "Epoch 92/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3395 - acc: 0.8535 - val_loss: 0.3892 - val_acc: 0.8270\n",
            "Epoch 93/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3376 - acc: 0.8529 - val_loss: 0.3898 - val_acc: 0.8246\n",
            "Epoch 94/500\n",
            "20109/20109 [==============================] - 12s 620us/sample - loss: 0.3375 - acc: 0.8523 - val_loss: 0.3884 - val_acc: 0.8266\n",
            "Epoch 95/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.3356 - acc: 0.8569 - val_loss: 0.3879 - val_acc: 0.8276\n",
            "Epoch 96/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3367 - acc: 0.8555 - val_loss: 0.3876 - val_acc: 0.8262\n",
            "Epoch 97/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.3363 - acc: 0.8561 - val_loss: 0.3869 - val_acc: 0.8280\n",
            "Epoch 98/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.3335 - acc: 0.8563 - val_loss: 0.3874 - val_acc: 0.8276\n",
            "Epoch 99/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.3329 - acc: 0.8576 - val_loss: 0.3868 - val_acc: 0.8284\n",
            "Epoch 100/500\n",
            "20109/20109 [==============================] - 13s 623us/sample - loss: 0.3312 - acc: 0.8591 - val_loss: 0.3863 - val_acc: 0.8280\n",
            "Epoch 101/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.3300 - acc: 0.8611 - val_loss: 0.3865 - val_acc: 0.8282\n",
            "Epoch 102/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3299 - acc: 0.8585 - val_loss: 0.3859 - val_acc: 0.8282\n",
            "Epoch 103/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.3264 - acc: 0.8597 - val_loss: 0.3855 - val_acc: 0.8288\n",
            "Epoch 104/500\n",
            "20109/20109 [==============================] - 12s 620us/sample - loss: 0.3259 - acc: 0.8602 - val_loss: 0.3848 - val_acc: 0.8288\n",
            "Epoch 105/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.3271 - acc: 0.8616 - val_loss: 0.3845 - val_acc: 0.8286\n",
            "Epoch 106/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3239 - acc: 0.8630 - val_loss: 0.3843 - val_acc: 0.8286\n",
            "Epoch 107/500\n",
            "20109/20109 [==============================] - 13s 624us/sample - loss: 0.3251 - acc: 0.8635 - val_loss: 0.3840 - val_acc: 0.8300\n",
            "Epoch 108/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.3227 - acc: 0.8648 - val_loss: 0.3840 - val_acc: 0.8286\n",
            "Epoch 109/500\n",
            "20109/20109 [==============================] - 13s 649us/sample - loss: 0.3222 - acc: 0.8631 - val_loss: 0.3836 - val_acc: 0.8284\n",
            "Epoch 110/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3197 - acc: 0.8642 - val_loss: 0.3830 - val_acc: 0.8304\n",
            "Epoch 111/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.3221 - acc: 0.8658 - val_loss: 0.3825 - val_acc: 0.8306\n",
            "Epoch 112/500\n",
            "20109/20109 [==============================] - 12s 602us/sample - loss: 0.3186 - acc: 0.8647 - val_loss: 0.3826 - val_acc: 0.8308\n",
            "Epoch 113/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3191 - acc: 0.8667 - val_loss: 0.3829 - val_acc: 0.8294\n",
            "Epoch 114/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.3137 - acc: 0.8679 - val_loss: 0.3825 - val_acc: 0.8300\n",
            "Epoch 115/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.3138 - acc: 0.8672 - val_loss: 0.3823 - val_acc: 0.8304\n",
            "Epoch 116/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3146 - acc: 0.8675 - val_loss: 0.3826 - val_acc: 0.8312\n",
            "Epoch 117/500\n",
            "20109/20109 [==============================] - 13s 622us/sample - loss: 0.3145 - acc: 0.8676 - val_loss: 0.3815 - val_acc: 0.8308\n",
            "Epoch 118/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.3109 - acc: 0.8700 - val_loss: 0.3817 - val_acc: 0.8316\n",
            "Epoch 119/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.3126 - acc: 0.8705 - val_loss: 0.3811 - val_acc: 0.8330\n",
            "Epoch 120/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.3090 - acc: 0.8709 - val_loss: 0.3814 - val_acc: 0.8322\n",
            "Epoch 121/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.3088 - acc: 0.8708 - val_loss: 0.3815 - val_acc: 0.8320\n",
            "Epoch 122/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.3080 - acc: 0.8711 - val_loss: 0.3813 - val_acc: 0.8328\n",
            "Epoch 123/500\n",
            "20109/20109 [==============================] - 12s 618us/sample - loss: 0.3119 - acc: 0.8706 - val_loss: 0.3810 - val_acc: 0.8334\n",
            "Epoch 124/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.3081 - acc: 0.8724 - val_loss: 0.3810 - val_acc: 0.8336\n",
            "Epoch 125/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.3053 - acc: 0.8723 - val_loss: 0.3815 - val_acc: 0.8342\n",
            "Epoch 126/500\n",
            "20109/20109 [==============================] - 12s 622us/sample - loss: 0.3082 - acc: 0.8735 - val_loss: 0.3809 - val_acc: 0.8340\n",
            "Epoch 127/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.3050 - acc: 0.8729 - val_loss: 0.3810 - val_acc: 0.8346\n",
            "Epoch 128/500\n",
            "20109/20109 [==============================] - 13s 646us/sample - loss: 0.3047 - acc: 0.8722 - val_loss: 0.3807 - val_acc: 0.8350\n",
            "Epoch 129/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3040 - acc: 0.8757 - val_loss: 0.3804 - val_acc: 0.8342\n",
            "Epoch 130/500\n",
            "20109/20109 [==============================] - 12s 614us/sample - loss: 0.3029 - acc: 0.8733 - val_loss: 0.3803 - val_acc: 0.8354\n",
            "Epoch 131/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.2997 - acc: 0.8769 - val_loss: 0.3808 - val_acc: 0.8356\n",
            "Epoch 132/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3001 - acc: 0.8766 - val_loss: 0.3805 - val_acc: 0.8372\n",
            "Epoch 133/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.2987 - acc: 0.8765 - val_loss: 0.3812 - val_acc: 0.8372\n",
            "Epoch 134/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.3006 - acc: 0.8742 - val_loss: 0.3799 - val_acc: 0.8358\n",
            "Epoch 135/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.3007 - acc: 0.8767 - val_loss: 0.3797 - val_acc: 0.8378\n",
            "Epoch 136/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.2961 - acc: 0.8778 - val_loss: 0.3801 - val_acc: 0.8356\n",
            "Epoch 137/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.2959 - acc: 0.8785 - val_loss: 0.3798 - val_acc: 0.8370\n",
            "Epoch 138/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.2955 - acc: 0.8791 - val_loss: 0.3804 - val_acc: 0.8372\n",
            "Epoch 139/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.2972 - acc: 0.8786 - val_loss: 0.3802 - val_acc: 0.8370\n",
            "Epoch 140/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.2944 - acc: 0.8794 - val_loss: 0.3797 - val_acc: 0.8362\n",
            "Epoch 141/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.2944 - acc: 0.8801 - val_loss: 0.3800 - val_acc: 0.8364\n",
            "Epoch 142/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.2932 - acc: 0.8801 - val_loss: 0.3802 - val_acc: 0.8368\n",
            "Epoch 143/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.2916 - acc: 0.8809 - val_loss: 0.3802 - val_acc: 0.8372\n",
            "Epoch 144/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.2899 - acc: 0.8825 - val_loss: 0.3798 - val_acc: 0.8364\n",
            "Epoch 145/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.2921 - acc: 0.8795 - val_loss: 0.3809 - val_acc: 0.8378\n",
            "Epoch 146/500\n",
            "20109/20109 [==============================] - 12s 619us/sample - loss: 0.2899 - acc: 0.8801 - val_loss: 0.3804 - val_acc: 0.8374\n",
            "Epoch 147/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.2912 - acc: 0.8794 - val_loss: 0.3799 - val_acc: 0.8374\n",
            "Epoch 00147: early stopping\n",
            "--- 1949.700116634369 seconds ---\n",
            "INFO:tensorflow:Copying TPU weights to the CPU\n",
            "20109/20109 [==============================] - 156s 8ms/sample - loss: 0.2709 - acc: 0.8896\n",
            "5027/5027 [==============================] - 38s 8ms/sample - loss: 0.3804 - acc: 0.8371\n",
            "Train Accuracy: 0.8896\n",
            "Train Loss: 0.2709\n",
            "Validation Accuracy:  0.8371\n",
            "Validation Loss: 0.3804\n",
            "5027/5027 [==============================] - 16s 3ms/sample\n",
            "[[1005  495]\n",
            " [ 324 3203]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.76      0.67      0.71      1500\n",
            "         1.0       0.87      0.91      0.89      3527\n",
            "\n",
            "    accuracy                           0.84      5027\n",
            "   macro avg       0.81      0.79      0.80      5027\n",
            "weighted avg       0.83      0.84      0.83      5027\n",
            "\n",
            "RMSE: 0.4036\n",
            "--- Fold 4 ---\n",
            "INFO:tensorflow:Querying Tensorflow master (grpc://10.126.17.194:8470) for TPU system metadata.\n",
            "INFO:tensorflow:Found TPU system:\n",
            "INFO:tensorflow:*** Num TPU Cores: 8\n",
            "INFO:tensorflow:*** Num TPU Workers: 1\n",
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 12028669822908458194)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, 8434022613342494329)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 9483302766372785140)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, 16913650255987291235)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, 9584223271632520063)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 5131928026390797549)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 3800337932879369162)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 2724390565822717472)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 1277980390101900352)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, 15973569101757891556)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 8589934592, 13938748725989702413)\n",
            "WARNING:tensorflow:tpu_model (from tensorflow.contrib.tpu.python.tpu.keras_support) is experimental and may change or be removed at any time, and without warning.\n",
            "Train on 20109 samples, validate on 5027 samples\n",
            "Epoch 1/500\n",
            "INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(128, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(128, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 9.07356309890747 secs\n",
            "INFO:tensorflow:Setting weights on TPU model.\n",
            "18432/20109 [==========================>...] - ETA: 3s - loss: 0.6925 - acc: 0.5864INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(81,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(81, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(81, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 11.853513240814209 secs\n",
            "19456/20109 [============================>.] - ETA: 1s - loss: 0.6925 - acc: 0.5914INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(128, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(128, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 10.14247989654541 secs\n",
            "INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(116,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(116, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(116, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 12.340025186538696 secs\n",
            "20109/20109 [==============================] - 94s 5ms/sample - loss: 0.6924 - acc: 0.5948 - val_loss: 0.6910 - val_acc: 0.7014\n",
            "Epoch 2/500\n",
            "20109/20109 [==============================] - 13s 622us/sample - loss: 0.6896 - acc: 0.6995 - val_loss: 0.6880 - val_acc: 0.7014\n",
            "Epoch 3/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.6863 - acc: 0.7017 - val_loss: 0.6842 - val_acc: 0.7014\n",
            "Epoch 4/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.6820 - acc: 0.7016 - val_loss: 0.6793 - val_acc: 0.7014\n",
            "Epoch 5/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.6763 - acc: 0.7017 - val_loss: 0.6728 - val_acc: 0.7014\n",
            "Epoch 6/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.6685 - acc: 0.7017 - val_loss: 0.6641 - val_acc: 0.7014\n",
            "Epoch 7/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.6587 - acc: 0.7017 - val_loss: 0.6528 - val_acc: 0.7014\n",
            "Epoch 8/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.6463 - acc: 0.7017 - val_loss: 0.6392 - val_acc: 0.7014\n",
            "Epoch 9/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.6327 - acc: 0.7017 - val_loss: 0.6257 - val_acc: 0.7014\n",
            "Epoch 10/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.6220 - acc: 0.7017 - val_loss: 0.6177 - val_acc: 0.7014\n",
            "Epoch 11/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.6165 - acc: 0.7017 - val_loss: 0.6140 - val_acc: 0.7014\n",
            "Epoch 12/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.6133 - acc: 0.7017 - val_loss: 0.6113 - val_acc: 0.7014\n",
            "Epoch 13/500\n",
            "20109/20109 [==============================] - 13s 646us/sample - loss: 0.6127 - acc: 0.7017 - val_loss: 0.6093 - val_acc: 0.7014\n",
            "Epoch 14/500\n",
            "20109/20109 [==============================] - 13s 623us/sample - loss: 0.6095 - acc: 0.7018 - val_loss: 0.6075 - val_acc: 0.7014\n",
            "Epoch 15/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.6076 - acc: 0.7017 - val_loss: 0.6057 - val_acc: 0.7014\n",
            "Epoch 16/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.6060 - acc: 0.7018 - val_loss: 0.6041 - val_acc: 0.7014\n",
            "Epoch 17/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.6044 - acc: 0.7017 - val_loss: 0.6024 - val_acc: 0.7014\n",
            "Epoch 18/500\n",
            "20109/20109 [==============================] - 12s 614us/sample - loss: 0.6025 - acc: 0.7017 - val_loss: 0.6006 - val_acc: 0.7014\n",
            "Epoch 19/500\n",
            "20109/20109 [==============================] - 13s 624us/sample - loss: 0.6015 - acc: 0.7017 - val_loss: 0.5988 - val_acc: 0.7014\n",
            "Epoch 20/500\n",
            "20109/20109 [==============================] - 13s 622us/sample - loss: 0.5980 - acc: 0.7017 - val_loss: 0.5966 - val_acc: 0.7014\n",
            "Epoch 21/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.5961 - acc: 0.7017 - val_loss: 0.5943 - val_acc: 0.7014\n",
            "Epoch 22/500\n",
            "20109/20109 [==============================] - 13s 644us/sample - loss: 0.5940 - acc: 0.7017 - val_loss: 0.5915 - val_acc: 0.7014\n",
            "Epoch 23/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.5909 - acc: 0.7016 - val_loss: 0.5885 - val_acc: 0.7014\n",
            "Epoch 24/500\n",
            "20109/20109 [==============================] - 13s 646us/sample - loss: 0.5891 - acc: 0.7017 - val_loss: 0.5849 - val_acc: 0.7014\n",
            "Epoch 25/500\n",
            "20109/20109 [==============================] - 13s 622us/sample - loss: 0.5840 - acc: 0.7016 - val_loss: 0.5806 - val_acc: 0.7014\n",
            "Epoch 26/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.5804 - acc: 0.7018 - val_loss: 0.5753 - val_acc: 0.7014\n",
            "Epoch 27/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.5738 - acc: 0.7017 - val_loss: 0.5690 - val_acc: 0.7014\n",
            "Epoch 28/500\n",
            "20109/20109 [==============================] - 13s 649us/sample - loss: 0.5685 - acc: 0.7014 - val_loss: 0.5625 - val_acc: 0.7014\n",
            "Epoch 29/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.5632 - acc: 0.7019 - val_loss: 0.5554 - val_acc: 0.7014\n",
            "Epoch 30/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.5533 - acc: 0.7039 - val_loss: 0.5452 - val_acc: 0.7022\n",
            "Epoch 31/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.5460 - acc: 0.7038 - val_loss: 0.5348 - val_acc: 0.7044\n",
            "Epoch 32/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.5371 - acc: 0.7103 - val_loss: 0.5244 - val_acc: 0.7116\n",
            "Epoch 33/500\n",
            "20109/20109 [==============================] - 12s 613us/sample - loss: 0.5260 - acc: 0.7201 - val_loss: 0.5117 - val_acc: 0.7213\n",
            "Epoch 34/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.5163 - acc: 0.7346 - val_loss: 0.5002 - val_acc: 0.7353\n",
            "Epoch 35/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.5070 - acc: 0.7431 - val_loss: 0.4896 - val_acc: 0.7520\n",
            "Epoch 36/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.5000 - acc: 0.7544 - val_loss: 0.4822 - val_acc: 0.7653\n",
            "Epoch 37/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.4915 - acc: 0.7611 - val_loss: 0.4741 - val_acc: 0.7699\n",
            "Epoch 38/500\n",
            "20109/20109 [==============================] - 13s 655us/sample - loss: 0.4863 - acc: 0.7673 - val_loss: 0.4674 - val_acc: 0.7741\n",
            "Epoch 39/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.4822 - acc: 0.7702 - val_loss: 0.4619 - val_acc: 0.7781\n",
            "Epoch 40/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.4758 - acc: 0.7732 - val_loss: 0.4570 - val_acc: 0.7830\n",
            "Epoch 41/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.4674 - acc: 0.7805 - val_loss: 0.4520 - val_acc: 0.7868\n",
            "Epoch 42/500\n",
            "20109/20109 [==============================] - 13s 645us/sample - loss: 0.4660 - acc: 0.7792 - val_loss: 0.4488 - val_acc: 0.7926\n",
            "Epoch 43/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.4566 - acc: 0.7872 - val_loss: 0.4445 - val_acc: 0.7920\n",
            "Epoch 44/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.4557 - acc: 0.7881 - val_loss: 0.4413 - val_acc: 0.7916\n",
            "Epoch 45/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.4481 - acc: 0.7927 - val_loss: 0.4377 - val_acc: 0.7932\n",
            "Epoch 46/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.4448 - acc: 0.7942 - val_loss: 0.4347 - val_acc: 0.7936\n",
            "Epoch 47/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.4384 - acc: 0.7970 - val_loss: 0.4319 - val_acc: 0.7964\n",
            "Epoch 48/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.4333 - acc: 0.8000 - val_loss: 0.4291 - val_acc: 0.7976\n",
            "Epoch 49/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.4318 - acc: 0.8028 - val_loss: 0.4271 - val_acc: 0.7994\n",
            "Epoch 50/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.4269 - acc: 0.8030 - val_loss: 0.4247 - val_acc: 0.8004\n",
            "Epoch 51/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.4197 - acc: 0.8087 - val_loss: 0.4216 - val_acc: 0.8014\n",
            "Epoch 52/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.4231 - acc: 0.8053 - val_loss: 0.4209 - val_acc: 0.8027\n",
            "Epoch 53/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.4190 - acc: 0.8082 - val_loss: 0.4192 - val_acc: 0.8045\n",
            "Epoch 54/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.4150 - acc: 0.8113 - val_loss: 0.4178 - val_acc: 0.8037\n",
            "Epoch 55/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.4164 - acc: 0.8091 - val_loss: 0.4163 - val_acc: 0.8045\n",
            "Epoch 56/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.4102 - acc: 0.8131 - val_loss: 0.4143 - val_acc: 0.8073\n",
            "Epoch 57/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.4051 - acc: 0.8146 - val_loss: 0.4126 - val_acc: 0.8085\n",
            "Epoch 58/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.4035 - acc: 0.8177 - val_loss: 0.4110 - val_acc: 0.8099\n",
            "Epoch 59/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3991 - acc: 0.8185 - val_loss: 0.4100 - val_acc: 0.8101\n",
            "Epoch 60/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.3962 - acc: 0.8192 - val_loss: 0.4081 - val_acc: 0.8115\n",
            "Epoch 61/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.3962 - acc: 0.8179 - val_loss: 0.4075 - val_acc: 0.8127\n",
            "Epoch 62/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.3944 - acc: 0.8201 - val_loss: 0.4068 - val_acc: 0.8125\n",
            "Epoch 63/500\n",
            "20109/20109 [==============================] - 13s 647us/sample - loss: 0.3919 - acc: 0.8225 - val_loss: 0.4059 - val_acc: 0.8139\n",
            "Epoch 64/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.3858 - acc: 0.8273 - val_loss: 0.4039 - val_acc: 0.8145\n",
            "Epoch 65/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3857 - acc: 0.8266 - val_loss: 0.4026 - val_acc: 0.8151\n",
            "Epoch 66/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.3828 - acc: 0.8300 - val_loss: 0.4016 - val_acc: 0.8167\n",
            "Epoch 67/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3838 - acc: 0.8274 - val_loss: 0.4007 - val_acc: 0.8185\n",
            "Epoch 68/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.3782 - acc: 0.8301 - val_loss: 0.3997 - val_acc: 0.8189\n",
            "Epoch 69/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.3775 - acc: 0.8311 - val_loss: 0.3992 - val_acc: 0.8189\n",
            "Epoch 70/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.3737 - acc: 0.8328 - val_loss: 0.3973 - val_acc: 0.8219\n",
            "Epoch 71/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.3747 - acc: 0.8300 - val_loss: 0.3964 - val_acc: 0.8229\n",
            "Epoch 72/500\n",
            "20109/20109 [==============================] - 13s 644us/sample - loss: 0.3710 - acc: 0.8349 - val_loss: 0.3960 - val_acc: 0.8211\n",
            "Epoch 73/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.3734 - acc: 0.8321 - val_loss: 0.3959 - val_acc: 0.8209\n",
            "Epoch 74/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.3674 - acc: 0.8367 - val_loss: 0.3953 - val_acc: 0.8217\n",
            "Epoch 75/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.3677 - acc: 0.8391 - val_loss: 0.3936 - val_acc: 0.8244\n",
            "Epoch 76/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.3628 - acc: 0.8395 - val_loss: 0.3935 - val_acc: 0.8234\n",
            "Epoch 77/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.3625 - acc: 0.8410 - val_loss: 0.3930 - val_acc: 0.8232\n",
            "Epoch 78/500\n",
            "20109/20109 [==============================] - 12s 620us/sample - loss: 0.3612 - acc: 0.8399 - val_loss: 0.3926 - val_acc: 0.8242\n",
            "Epoch 79/500\n",
            "20109/20109 [==============================] - 13s 644us/sample - loss: 0.3590 - acc: 0.8411 - val_loss: 0.3922 - val_acc: 0.8242\n",
            "Epoch 80/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.3564 - acc: 0.8418 - val_loss: 0.3900 - val_acc: 0.8256\n",
            "Epoch 81/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.3550 - acc: 0.8437 - val_loss: 0.3894 - val_acc: 0.8252\n",
            "Epoch 82/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.3530 - acc: 0.8458 - val_loss: 0.3893 - val_acc: 0.8252\n",
            "Epoch 83/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3527 - acc: 0.8458 - val_loss: 0.3886 - val_acc: 0.8262\n",
            "Epoch 84/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.3519 - acc: 0.8445 - val_loss: 0.3879 - val_acc: 0.8274\n",
            "Epoch 85/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.3512 - acc: 0.8461 - val_loss: 0.3874 - val_acc: 0.8274\n",
            "Epoch 86/500\n",
            "20109/20109 [==============================] - 13s 646us/sample - loss: 0.3455 - acc: 0.8501 - val_loss: 0.3867 - val_acc: 0.8298\n",
            "Epoch 87/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3462 - acc: 0.8513 - val_loss: 0.3864 - val_acc: 0.8296\n",
            "Epoch 88/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.3470 - acc: 0.8457 - val_loss: 0.3866 - val_acc: 0.8290\n",
            "Epoch 89/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.3442 - acc: 0.8504 - val_loss: 0.3853 - val_acc: 0.8302\n",
            "Epoch 90/500\n",
            "20109/20109 [==============================] - 13s 646us/sample - loss: 0.3425 - acc: 0.8501 - val_loss: 0.3846 - val_acc: 0.8308\n",
            "Epoch 91/500\n",
            "20109/20109 [==============================] - 13s 646us/sample - loss: 0.3431 - acc: 0.8491 - val_loss: 0.3852 - val_acc: 0.8284\n",
            "Epoch 92/500\n",
            "20109/20109 [==============================] - 13s 622us/sample - loss: 0.3400 - acc: 0.8511 - val_loss: 0.3835 - val_acc: 0.8324\n",
            "Epoch 93/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.3356 - acc: 0.8535 - val_loss: 0.3831 - val_acc: 0.8330\n",
            "Epoch 94/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.3373 - acc: 0.8531 - val_loss: 0.3827 - val_acc: 0.8326\n",
            "Epoch 95/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3387 - acc: 0.8539 - val_loss: 0.3822 - val_acc: 0.8324\n",
            "Epoch 96/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3348 - acc: 0.8524 - val_loss: 0.3836 - val_acc: 0.8298\n",
            "Epoch 97/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.3346 - acc: 0.8552 - val_loss: 0.3819 - val_acc: 0.8318\n",
            "Epoch 98/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3335 - acc: 0.8562 - val_loss: 0.3810 - val_acc: 0.8338\n",
            "Epoch 99/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.3331 - acc: 0.8575 - val_loss: 0.3804 - val_acc: 0.8340\n",
            "Epoch 100/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3321 - acc: 0.8563 - val_loss: 0.3799 - val_acc: 0.8338\n",
            "Epoch 101/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3311 - acc: 0.8567 - val_loss: 0.3791 - val_acc: 0.8338\n",
            "Epoch 102/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.3265 - acc: 0.8595 - val_loss: 0.3792 - val_acc: 0.8344\n",
            "Epoch 103/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.3268 - acc: 0.8599 - val_loss: 0.3784 - val_acc: 0.8344\n",
            "Epoch 104/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.3274 - acc: 0.8589 - val_loss: 0.3790 - val_acc: 0.8336\n",
            "Epoch 105/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.3244 - acc: 0.8600 - val_loss: 0.3785 - val_acc: 0.8340\n",
            "Epoch 106/500\n",
            "20109/20109 [==============================] - 13s 644us/sample - loss: 0.3248 - acc: 0.8605 - val_loss: 0.3784 - val_acc: 0.8342\n",
            "Epoch 107/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.3248 - acc: 0.8625 - val_loss: 0.3781 - val_acc: 0.8342\n",
            "Epoch 108/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.3226 - acc: 0.8627 - val_loss: 0.3783 - val_acc: 0.8352\n",
            "Epoch 109/500\n",
            "20109/20109 [==============================] - 13s 648us/sample - loss: 0.3195 - acc: 0.8639 - val_loss: 0.3774 - val_acc: 0.8366\n",
            "Epoch 110/500\n",
            "20109/20109 [==============================] - 12s 619us/sample - loss: 0.3188 - acc: 0.8635 - val_loss: 0.3774 - val_acc: 0.8364\n",
            "Epoch 111/500\n",
            "20109/20109 [==============================] - 13s 658us/sample - loss: 0.3182 - acc: 0.8660 - val_loss: 0.3767 - val_acc: 0.8370\n",
            "Epoch 112/500\n",
            "20109/20109 [==============================] - 13s 646us/sample - loss: 0.3186 - acc: 0.8663 - val_loss: 0.3758 - val_acc: 0.8368\n",
            "Epoch 113/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.3173 - acc: 0.8639 - val_loss: 0.3769 - val_acc: 0.8362\n",
            "Epoch 114/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3182 - acc: 0.8648 - val_loss: 0.3759 - val_acc: 0.8376\n",
            "Epoch 115/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.3184 - acc: 0.8640 - val_loss: 0.3763 - val_acc: 0.8364\n",
            "Epoch 116/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.3133 - acc: 0.8679 - val_loss: 0.3765 - val_acc: 0.8370\n",
            "Epoch 117/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.3129 - acc: 0.8684 - val_loss: 0.3758 - val_acc: 0.8378\n",
            "Epoch 118/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.3113 - acc: 0.8678 - val_loss: 0.3753 - val_acc: 0.8374\n",
            "Epoch 119/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.3118 - acc: 0.8664 - val_loss: 0.3756 - val_acc: 0.8382\n",
            "Epoch 120/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3108 - acc: 0.8695 - val_loss: 0.3746 - val_acc: 0.8384\n",
            "Epoch 121/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.3122 - acc: 0.8700 - val_loss: 0.3741 - val_acc: 0.8378\n",
            "Epoch 122/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.3137 - acc: 0.8682 - val_loss: 0.3743 - val_acc: 0.8380\n",
            "Epoch 123/500\n",
            "20109/20109 [==============================] - 13s 644us/sample - loss: 0.3116 - acc: 0.8683 - val_loss: 0.3739 - val_acc: 0.8388\n",
            "Epoch 124/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.3075 - acc: 0.8717 - val_loss: 0.3743 - val_acc: 0.8398\n",
            "Epoch 125/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.3078 - acc: 0.8711 - val_loss: 0.3729 - val_acc: 0.8384\n",
            "Epoch 126/500\n",
            "20109/20109 [==============================] - 12s 614us/sample - loss: 0.3060 - acc: 0.8706 - val_loss: 0.3727 - val_acc: 0.8386\n",
            "Epoch 127/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.3035 - acc: 0.8719 - val_loss: 0.3731 - val_acc: 0.8390\n",
            "Epoch 128/500\n",
            "20109/20109 [==============================] - 13s 644us/sample - loss: 0.3045 - acc: 0.8730 - val_loss: 0.3738 - val_acc: 0.8392\n",
            "Epoch 129/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.3021 - acc: 0.8748 - val_loss: 0.3733 - val_acc: 0.8390\n",
            "Epoch 130/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3029 - acc: 0.8744 - val_loss: 0.3723 - val_acc: 0.8394\n",
            "Epoch 131/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.3036 - acc: 0.8739 - val_loss: 0.3724 - val_acc: 0.8394\n",
            "Epoch 132/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3037 - acc: 0.8716 - val_loss: 0.3722 - val_acc: 0.8396\n",
            "Epoch 133/500\n",
            "20109/20109 [==============================] - 12s 617us/sample - loss: 0.3011 - acc: 0.8749 - val_loss: 0.3716 - val_acc: 0.8408\n",
            "Epoch 134/500\n",
            "20109/20109 [==============================] - 12s 621us/sample - loss: 0.3013 - acc: 0.8739 - val_loss: 0.3722 - val_acc: 0.8390\n",
            "Epoch 135/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.2993 - acc: 0.8765 - val_loss: 0.3718 - val_acc: 0.8398\n",
            "Epoch 136/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.2957 - acc: 0.8758 - val_loss: 0.3727 - val_acc: 0.8392\n",
            "Epoch 137/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.2999 - acc: 0.8759 - val_loss: 0.3721 - val_acc: 0.8406\n",
            "Epoch 138/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.2960 - acc: 0.8755 - val_loss: 0.3716 - val_acc: 0.8410\n",
            "Epoch 139/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.2973 - acc: 0.8782 - val_loss: 0.3721 - val_acc: 0.8404\n",
            "Epoch 140/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.2981 - acc: 0.8769 - val_loss: 0.3716 - val_acc: 0.8398\n",
            "Epoch 141/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.2952 - acc: 0.8776 - val_loss: 0.3724 - val_acc: 0.8398\n",
            "Epoch 142/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.2939 - acc: 0.8781 - val_loss: 0.3719 - val_acc: 0.8408\n",
            "Epoch 143/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.2928 - acc: 0.8792 - val_loss: 0.3714 - val_acc: 0.8404\n",
            "Epoch 144/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.2921 - acc: 0.8776 - val_loss: 0.3715 - val_acc: 0.8402\n",
            "Epoch 145/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.2966 - acc: 0.8760 - val_loss: 0.3713 - val_acc: 0.8396\n",
            "Epoch 146/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.2929 - acc: 0.8789 - val_loss: 0.3719 - val_acc: 0.8410\n",
            "Epoch 147/500\n",
            "20109/20109 [==============================] - 13s 644us/sample - loss: 0.2926 - acc: 0.8783 - val_loss: 0.3710 - val_acc: 0.8420\n",
            "Epoch 148/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.2898 - acc: 0.8787 - val_loss: 0.3711 - val_acc: 0.8418\n",
            "Epoch 149/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.2890 - acc: 0.8817 - val_loss: 0.3716 - val_acc: 0.8414\n",
            "Epoch 150/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.2902 - acc: 0.8783 - val_loss: 0.3699 - val_acc: 0.8408\n",
            "Epoch 151/500\n",
            "20109/20109 [==============================] - 13s 623us/sample - loss: 0.2912 - acc: 0.8808 - val_loss: 0.3700 - val_acc: 0.8412\n",
            "Epoch 152/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.2877 - acc: 0.8821 - val_loss: 0.3705 - val_acc: 0.8428\n",
            "Epoch 153/500\n",
            "20109/20109 [==============================] - 13s 645us/sample - loss: 0.2891 - acc: 0.8803 - val_loss: 0.3706 - val_acc: 0.8418\n",
            "Epoch 154/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.2875 - acc: 0.8807 - val_loss: 0.3704 - val_acc: 0.8416\n",
            "Epoch 155/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.2842 - acc: 0.8855 - val_loss: 0.3707 - val_acc: 0.8430\n",
            "Epoch 156/500\n",
            "20109/20109 [==============================] - 12s 616us/sample - loss: 0.2871 - acc: 0.8817 - val_loss: 0.3702 - val_acc: 0.8428\n",
            "Epoch 157/500\n",
            "20109/20109 [==============================] - 12s 620us/sample - loss: 0.2877 - acc: 0.8818 - val_loss: 0.3697 - val_acc: 0.8428\n",
            "Epoch 158/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.2831 - acc: 0.8834 - val_loss: 0.3706 - val_acc: 0.8420\n",
            "Epoch 159/500\n",
            "20109/20109 [==============================] - 13s 654us/sample - loss: 0.2866 - acc: 0.8833 - val_loss: 0.3695 - val_acc: 0.8432\n",
            "Epoch 160/500\n",
            "20109/20109 [==============================] - 12s 619us/sample - loss: 0.2847 - acc: 0.8840 - val_loss: 0.3704 - val_acc: 0.8434\n",
            "Epoch 161/500\n",
            "20109/20109 [==============================] - 13s 646us/sample - loss: 0.2833 - acc: 0.8847 - val_loss: 0.3704 - val_acc: 0.8441\n",
            "Epoch 162/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.2846 - acc: 0.8820 - val_loss: 0.3692 - val_acc: 0.8439\n",
            "Epoch 163/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.2841 - acc: 0.8843 - val_loss: 0.3691 - val_acc: 0.8434\n",
            "Epoch 164/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.2814 - acc: 0.8856 - val_loss: 0.3702 - val_acc: 0.8436\n",
            "Epoch 165/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.2832 - acc: 0.8858 - val_loss: 0.3703 - val_acc: 0.8436\n",
            "Epoch 166/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.2804 - acc: 0.8847 - val_loss: 0.3700 - val_acc: 0.8447\n",
            "Epoch 167/500\n",
            "20109/20109 [==============================] - 13s 646us/sample - loss: 0.2824 - acc: 0.8852 - val_loss: 0.3700 - val_acc: 0.8439\n",
            "Epoch 168/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.2799 - acc: 0.8883 - val_loss: 0.3693 - val_acc: 0.8438\n",
            "Epoch 169/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.2774 - acc: 0.8844 - val_loss: 0.3692 - val_acc: 0.8441\n",
            "Epoch 170/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.2799 - acc: 0.8872 - val_loss: 0.3701 - val_acc: 0.8445\n",
            "Epoch 00170: early stopping\n",
            "--- 2252.494204044342 seconds ---\n",
            "INFO:tensorflow:Copying TPU weights to the CPU\n",
            "20109/20109 [==============================] - 162s 8ms/sample - loss: 0.2604 - acc: 0.8950\n",
            "5027/5027 [==============================] - 41s 8ms/sample - loss: 0.3700 - acc: 0.8446\n",
            "Train Accuracy: 0.8950\n",
            "Train Loss: 0.2604\n",
            "Validation Accuracy:  0.8446\n",
            "Validation Loss: 0.3700\n",
            "5027/5027 [==============================] - 15s 3ms/sample\n",
            "[[1018  482]\n",
            " [ 299 3228]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.77      0.68      0.72      1500\n",
            "         1.0       0.87      0.92      0.89      3527\n",
            "\n",
            "    accuracy                           0.84      5027\n",
            "   macro avg       0.82      0.80      0.81      5027\n",
            "weighted avg       0.84      0.84      0.84      5027\n",
            "\n",
            "RMSE: 0.3942\n",
            "--- Fold 5 ---\n",
            "INFO:tensorflow:Querying Tensorflow master (grpc://10.126.17.194:8470) for TPU system metadata.\n",
            "INFO:tensorflow:Found TPU system:\n",
            "INFO:tensorflow:*** Num TPU Cores: 8\n",
            "INFO:tensorflow:*** Num TPU Workers: 1\n",
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 12028669822908458194)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, 8434022613342494329)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 9483302766372785140)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, 16913650255987291235)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, 9584223271632520063)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 5131928026390797549)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 3800337932879369162)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 2724390565822717472)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 1277980390101900352)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, 15973569101757891556)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 8589934592, 13938748725989702413)\n",
            "WARNING:tensorflow:tpu_model (from tensorflow.contrib.tpu.python.tpu.keras_support) is experimental and may change or be removed at any time, and without warning.\n",
            "Train on 20109 samples, validate on 5027 samples\n",
            "Epoch 1/500\n",
            "INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(128, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(128, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 9.400815486907959 secs\n",
            "INFO:tensorflow:Setting weights on TPU model.\n",
            "18432/20109 [==========================>...] - ETA: 3s - loss: 0.6915 - acc: 0.6357INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(81,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(81, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(81, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 11.843267440795898 secs\n",
            "19456/20109 [============================>.] - ETA: 1s - loss: 0.6915 - acc: 0.6390INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(128, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(128, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 10.474747657775879 secs\n",
            "INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(116,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(116, 500), dtype=tf.float32, name='Input_10'), TensorSpec(shape=(116, 1), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for Input\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 12.12550163269043 secs\n",
            "20109/20109 [==============================] - 93s 5ms/sample - loss: 0.6914 - acc: 0.6414 - val_loss: 0.6895 - val_acc: 0.7018\n",
            "Epoch 2/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.6876 - acc: 0.7013 - val_loss: 0.6854 - val_acc: 0.7018\n",
            "Epoch 3/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.6829 - acc: 0.7017 - val_loss: 0.6800 - val_acc: 0.7018\n",
            "Epoch 4/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.6766 - acc: 0.7017 - val_loss: 0.6728 - val_acc: 0.7018\n",
            "Epoch 5/500\n",
            "20109/20109 [==============================] - 13s 661us/sample - loss: 0.6683 - acc: 0.7017 - val_loss: 0.6633 - val_acc: 0.7018\n",
            "Epoch 6/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.6573 - acc: 0.7017 - val_loss: 0.6512 - val_acc: 0.7018\n",
            "Epoch 7/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.6442 - acc: 0.7017 - val_loss: 0.6372 - val_acc: 0.7018\n",
            "Epoch 8/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.6313 - acc: 0.7017 - val_loss: 0.6257 - val_acc: 0.7018\n",
            "Epoch 9/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.6223 - acc: 0.7016 - val_loss: 0.6193 - val_acc: 0.7018\n",
            "Epoch 10/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.6188 - acc: 0.7017 - val_loss: 0.6156 - val_acc: 0.7018\n",
            "Epoch 11/500\n",
            "20109/20109 [==============================] - 13s 644us/sample - loss: 0.6163 - acc: 0.7016 - val_loss: 0.6126 - val_acc: 0.7018\n",
            "Epoch 12/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.6131 - acc: 0.7018 - val_loss: 0.6103 - val_acc: 0.7018\n",
            "Epoch 13/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.6112 - acc: 0.7018 - val_loss: 0.6083 - val_acc: 0.7018\n",
            "Epoch 14/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.6081 - acc: 0.7016 - val_loss: 0.6066 - val_acc: 0.7018\n",
            "Epoch 15/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.6079 - acc: 0.7017 - val_loss: 0.6051 - val_acc: 0.7018\n",
            "Epoch 16/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.6049 - acc: 0.7017 - val_loss: 0.6036 - val_acc: 0.7018\n",
            "Epoch 17/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.6041 - acc: 0.7017 - val_loss: 0.6019 - val_acc: 0.7018\n",
            "Epoch 18/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.6037 - acc: 0.7017 - val_loss: 0.6004 - val_acc: 0.7018\n",
            "Epoch 19/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.6010 - acc: 0.7017 - val_loss: 0.5985 - val_acc: 0.7018\n",
            "Epoch 20/500\n",
            "20109/20109 [==============================] - 13s 652us/sample - loss: 0.5997 - acc: 0.7017 - val_loss: 0.5966 - val_acc: 0.7018\n",
            "Epoch 21/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.5965 - acc: 0.7017 - val_loss: 0.5943 - val_acc: 0.7018\n",
            "Epoch 22/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.5945 - acc: 0.7017 - val_loss: 0.5916 - val_acc: 0.7018\n",
            "Epoch 23/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.5917 - acc: 0.7017 - val_loss: 0.5883 - val_acc: 0.7018\n",
            "Epoch 24/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.5884 - acc: 0.7017 - val_loss: 0.5849 - val_acc: 0.7018\n",
            "Epoch 25/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.5841 - acc: 0.7018 - val_loss: 0.5806 - val_acc: 0.7018\n",
            "Epoch 26/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.5791 - acc: 0.7018 - val_loss: 0.5753 - val_acc: 0.7018\n",
            "Epoch 27/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.5745 - acc: 0.7017 - val_loss: 0.5690 - val_acc: 0.7018\n",
            "Epoch 28/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.5685 - acc: 0.7019 - val_loss: 0.5615 - val_acc: 0.7018\n",
            "Epoch 29/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.5606 - acc: 0.7023 - val_loss: 0.5533 - val_acc: 0.7022\n",
            "Epoch 30/500\n",
            "20109/20109 [==============================] - 13s 624us/sample - loss: 0.5520 - acc: 0.7026 - val_loss: 0.5427 - val_acc: 0.7026\n",
            "Epoch 31/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.5439 - acc: 0.7079 - val_loss: 0.5311 - val_acc: 0.7056\n",
            "Epoch 32/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.5356 - acc: 0.7142 - val_loss: 0.5194 - val_acc: 0.7142\n",
            "Epoch 33/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.5240 - acc: 0.7235 - val_loss: 0.5077 - val_acc: 0.7269\n",
            "Epoch 34/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.5141 - acc: 0.7387 - val_loss: 0.4949 - val_acc: 0.7432\n",
            "Epoch 35/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.5080 - acc: 0.7481 - val_loss: 0.4856 - val_acc: 0.7570\n",
            "Epoch 36/500\n",
            "20109/20109 [==============================] - 13s 651us/sample - loss: 0.4949 - acc: 0.7571 - val_loss: 0.4750 - val_acc: 0.7617\n",
            "Epoch 37/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.4880 - acc: 0.7708 - val_loss: 0.4675 - val_acc: 0.7677\n",
            "Epoch 38/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.4860 - acc: 0.7714 - val_loss: 0.4629 - val_acc: 0.7717\n",
            "Epoch 39/500\n",
            "20109/20109 [==============================] - 12s 620us/sample - loss: 0.4773 - acc: 0.7774 - val_loss: 0.4563 - val_acc: 0.7775\n",
            "Epoch 40/500\n",
            "20109/20109 [==============================] - 13s 648us/sample - loss: 0.4749 - acc: 0.7767 - val_loss: 0.4520 - val_acc: 0.7803\n",
            "Epoch 41/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.4698 - acc: 0.7787 - val_loss: 0.4486 - val_acc: 0.7832\n",
            "Epoch 42/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.4629 - acc: 0.7847 - val_loss: 0.4442 - val_acc: 0.7842\n",
            "Epoch 43/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.4543 - acc: 0.7893 - val_loss: 0.4404 - val_acc: 0.7890\n",
            "Epoch 44/500\n",
            "20109/20109 [==============================] - 13s 644us/sample - loss: 0.4516 - acc: 0.7919 - val_loss: 0.4365 - val_acc: 0.7888\n",
            "Epoch 45/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.4443 - acc: 0.7971 - val_loss: 0.4330 - val_acc: 0.7894\n",
            "Epoch 46/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.4431 - acc: 0.7978 - val_loss: 0.4314 - val_acc: 0.7938\n",
            "Epoch 47/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.4373 - acc: 0.8000 - val_loss: 0.4286 - val_acc: 0.7944\n",
            "Epoch 48/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.4315 - acc: 0.8021 - val_loss: 0.4250 - val_acc: 0.7952\n",
            "Epoch 49/500\n",
            "20109/20109 [==============================] - 12s 614us/sample - loss: 0.4308 - acc: 0.8040 - val_loss: 0.4236 - val_acc: 0.7948\n",
            "Epoch 50/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.4251 - acc: 0.8043 - val_loss: 0.4220 - val_acc: 0.7954\n",
            "Epoch 51/500\n",
            "20109/20109 [==============================] - 13s 648us/sample - loss: 0.4219 - acc: 0.8066 - val_loss: 0.4201 - val_acc: 0.7964\n",
            "Epoch 52/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.4238 - acc: 0.8074 - val_loss: 0.4183 - val_acc: 0.8000\n",
            "Epoch 53/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.4170 - acc: 0.8109 - val_loss: 0.4162 - val_acc: 0.7984\n",
            "Epoch 54/500\n",
            "20109/20109 [==============================] - 13s 646us/sample - loss: 0.4164 - acc: 0.8113 - val_loss: 0.4148 - val_acc: 0.8010\n",
            "Epoch 55/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.4113 - acc: 0.8122 - val_loss: 0.4140 - val_acc: 0.8000\n",
            "Epoch 56/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.4045 - acc: 0.8160 - val_loss: 0.4113 - val_acc: 0.8021\n",
            "Epoch 57/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.4057 - acc: 0.8154 - val_loss: 0.4101 - val_acc: 0.8045\n",
            "Epoch 58/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.4005 - acc: 0.8194 - val_loss: 0.4083 - val_acc: 0.8055\n",
            "Epoch 59/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3975 - acc: 0.8213 - val_loss: 0.4069 - val_acc: 0.8081\n",
            "Epoch 60/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3944 - acc: 0.8211 - val_loss: 0.4054 - val_acc: 0.8083\n",
            "Epoch 61/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.3956 - acc: 0.8242 - val_loss: 0.4043 - val_acc: 0.8079\n",
            "Epoch 62/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.3926 - acc: 0.8245 - val_loss: 0.4032 - val_acc: 0.8089\n",
            "Epoch 63/500\n",
            "20109/20109 [==============================] - 13s 651us/sample - loss: 0.3893 - acc: 0.8239 - val_loss: 0.4019 - val_acc: 0.8097\n",
            "Epoch 64/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.3890 - acc: 0.8239 - val_loss: 0.4011 - val_acc: 0.8089\n",
            "Epoch 65/500\n",
            "20109/20109 [==============================] - 13s 623us/sample - loss: 0.3852 - acc: 0.8284 - val_loss: 0.3994 - val_acc: 0.8113\n",
            "Epoch 66/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.3870 - acc: 0.8242 - val_loss: 0.3986 - val_acc: 0.8105\n",
            "Epoch 67/500\n",
            "20109/20109 [==============================] - 12s 621us/sample - loss: 0.3788 - acc: 0.8313 - val_loss: 0.3969 - val_acc: 0.8139\n",
            "Epoch 68/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.3802 - acc: 0.8294 - val_loss: 0.3961 - val_acc: 0.8127\n",
            "Epoch 69/500\n",
            "20109/20109 [==============================] - 13s 657us/sample - loss: 0.3780 - acc: 0.8322 - val_loss: 0.3953 - val_acc: 0.8125\n",
            "Epoch 70/500\n",
            "20109/20109 [==============================] - 13s 653us/sample - loss: 0.3772 - acc: 0.8312 - val_loss: 0.3943 - val_acc: 0.8135\n",
            "Epoch 71/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.3728 - acc: 0.8350 - val_loss: 0.3928 - val_acc: 0.8145\n",
            "Epoch 72/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.3719 - acc: 0.8350 - val_loss: 0.3920 - val_acc: 0.8169\n",
            "Epoch 73/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.3709 - acc: 0.8362 - val_loss: 0.3912 - val_acc: 0.8163\n",
            "Epoch 74/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.3686 - acc: 0.8365 - val_loss: 0.3908 - val_acc: 0.8157\n",
            "Epoch 75/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.3655 - acc: 0.8381 - val_loss: 0.3895 - val_acc: 0.8179\n",
            "Epoch 76/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.3633 - acc: 0.8392 - val_loss: 0.3883 - val_acc: 0.8199\n",
            "Epoch 77/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.3598 - acc: 0.8441 - val_loss: 0.3876 - val_acc: 0.8205\n",
            "Epoch 78/500\n",
            "20109/20109 [==============================] - 12s 622us/sample - loss: 0.3622 - acc: 0.8390 - val_loss: 0.3874 - val_acc: 0.8193\n",
            "Epoch 79/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.3563 - acc: 0.8448 - val_loss: 0.3861 - val_acc: 0.8209\n",
            "Epoch 80/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.3567 - acc: 0.8444 - val_loss: 0.3855 - val_acc: 0.8217\n",
            "Epoch 81/500\n",
            "20109/20109 [==============================] - 13s 650us/sample - loss: 0.3561 - acc: 0.8426 - val_loss: 0.3841 - val_acc: 0.8223\n",
            "Epoch 82/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.3570 - acc: 0.8425 - val_loss: 0.3839 - val_acc: 0.8221\n",
            "Epoch 83/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.3564 - acc: 0.8449 - val_loss: 0.3827 - val_acc: 0.8227\n",
            "Epoch 84/500\n",
            "20109/20109 [==============================] - 13s 635us/sample - loss: 0.3505 - acc: 0.8480 - val_loss: 0.3820 - val_acc: 0.8238\n",
            "Epoch 85/500\n",
            "20109/20109 [==============================] - 13s 623us/sample - loss: 0.3491 - acc: 0.8486 - val_loss: 0.3815 - val_acc: 0.8225\n",
            "Epoch 86/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.3490 - acc: 0.8489 - val_loss: 0.3811 - val_acc: 0.8234\n",
            "Epoch 87/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.3481 - acc: 0.8485 - val_loss: 0.3808 - val_acc: 0.8238\n",
            "Epoch 88/500\n",
            "20109/20109 [==============================] - 13s 647us/sample - loss: 0.3462 - acc: 0.8502 - val_loss: 0.3795 - val_acc: 0.8248\n",
            "Epoch 89/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.3455 - acc: 0.8494 - val_loss: 0.3787 - val_acc: 0.8256\n",
            "Epoch 90/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3426 - acc: 0.8512 - val_loss: 0.3781 - val_acc: 0.8262\n",
            "Epoch 91/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3408 - acc: 0.8534 - val_loss: 0.3785 - val_acc: 0.8248\n",
            "Epoch 92/500\n",
            "20109/20109 [==============================] - 12s 621us/sample - loss: 0.3415 - acc: 0.8532 - val_loss: 0.3772 - val_acc: 0.8256\n",
            "Epoch 93/500\n",
            "20109/20109 [==============================] - 13s 644us/sample - loss: 0.3404 - acc: 0.8518 - val_loss: 0.3769 - val_acc: 0.8262\n",
            "Epoch 94/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.3390 - acc: 0.8536 - val_loss: 0.3767 - val_acc: 0.8254\n",
            "Epoch 95/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.3379 - acc: 0.8547 - val_loss: 0.3752 - val_acc: 0.8272\n",
            "Epoch 96/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.3365 - acc: 0.8553 - val_loss: 0.3747 - val_acc: 0.8286\n",
            "Epoch 97/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.3350 - acc: 0.8558 - val_loss: 0.3742 - val_acc: 0.8298\n",
            "Epoch 98/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.3340 - acc: 0.8557 - val_loss: 0.3739 - val_acc: 0.8294\n",
            "Epoch 99/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.3333 - acc: 0.8611 - val_loss: 0.3733 - val_acc: 0.8302\n",
            "Epoch 100/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3293 - acc: 0.8592 - val_loss: 0.3729 - val_acc: 0.8310\n",
            "Epoch 101/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.3321 - acc: 0.8574 - val_loss: 0.3726 - val_acc: 0.8304\n",
            "Epoch 102/500\n",
            "20109/20109 [==============================] - 12s 619us/sample - loss: 0.3313 - acc: 0.8583 - val_loss: 0.3722 - val_acc: 0.8302\n",
            "Epoch 103/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3272 - acc: 0.8609 - val_loss: 0.3712 - val_acc: 0.8322\n",
            "Epoch 104/500\n",
            "20109/20109 [==============================] - 13s 650us/sample - loss: 0.3278 - acc: 0.8601 - val_loss: 0.3710 - val_acc: 0.8320\n",
            "Epoch 105/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.3244 - acc: 0.8616 - val_loss: 0.3705 - val_acc: 0.8320\n",
            "Epoch 106/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.3288 - acc: 0.8601 - val_loss: 0.3697 - val_acc: 0.8330\n",
            "Epoch 107/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3239 - acc: 0.8608 - val_loss: 0.3700 - val_acc: 0.8314\n",
            "Epoch 108/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.3239 - acc: 0.8599 - val_loss: 0.3696 - val_acc: 0.8324\n",
            "Epoch 109/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.3221 - acc: 0.8641 - val_loss: 0.3694 - val_acc: 0.8330\n",
            "Epoch 110/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.3194 - acc: 0.8656 - val_loss: 0.3689 - val_acc: 0.8338\n",
            "Epoch 111/500\n",
            "20109/20109 [==============================] - 13s 654us/sample - loss: 0.3195 - acc: 0.8641 - val_loss: 0.3681 - val_acc: 0.8358\n",
            "Epoch 112/500\n",
            "20109/20109 [==============================] - 13s 646us/sample - loss: 0.3181 - acc: 0.8652 - val_loss: 0.3679 - val_acc: 0.8350\n",
            "Epoch 113/500\n",
            "20109/20109 [==============================] - 13s 622us/sample - loss: 0.3183 - acc: 0.8670 - val_loss: 0.3677 - val_acc: 0.8356\n",
            "Epoch 114/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.3168 - acc: 0.8658 - val_loss: 0.3673 - val_acc: 0.8356\n",
            "Epoch 115/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3173 - acc: 0.8650 - val_loss: 0.3671 - val_acc: 0.8352\n",
            "Epoch 116/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3149 - acc: 0.8669 - val_loss: 0.3662 - val_acc: 0.8354\n",
            "Epoch 117/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3120 - acc: 0.8681 - val_loss: 0.3657 - val_acc: 0.8362\n",
            "Epoch 118/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.3131 - acc: 0.8672 - val_loss: 0.3663 - val_acc: 0.8338\n",
            "Epoch 119/500\n",
            "20109/20109 [==============================] - 13s 624us/sample - loss: 0.3119 - acc: 0.8678 - val_loss: 0.3656 - val_acc: 0.8354\n",
            "Epoch 120/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.3147 - acc: 0.8685 - val_loss: 0.3649 - val_acc: 0.8360\n",
            "Epoch 121/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.3120 - acc: 0.8673 - val_loss: 0.3649 - val_acc: 0.8364\n",
            "Epoch 122/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.3096 - acc: 0.8682 - val_loss: 0.3645 - val_acc: 0.8368\n",
            "Epoch 123/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3101 - acc: 0.8693 - val_loss: 0.3637 - val_acc: 0.8374\n",
            "Epoch 124/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.3095 - acc: 0.8702 - val_loss: 0.3640 - val_acc: 0.8372\n",
            "Epoch 125/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.3069 - acc: 0.8707 - val_loss: 0.3636 - val_acc: 0.8384\n",
            "Epoch 126/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.3052 - acc: 0.8703 - val_loss: 0.3633 - val_acc: 0.8378\n",
            "Epoch 127/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.3039 - acc: 0.8723 - val_loss: 0.3646 - val_acc: 0.8370\n",
            "Epoch 128/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.3102 - acc: 0.8712 - val_loss: 0.3631 - val_acc: 0.8386\n",
            "Epoch 129/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.3067 - acc: 0.8730 - val_loss: 0.3639 - val_acc: 0.8368\n",
            "Epoch 130/500\n",
            "20109/20109 [==============================] - 13s 625us/sample - loss: 0.3034 - acc: 0.8728 - val_loss: 0.3628 - val_acc: 0.8382\n",
            "Epoch 131/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.3033 - acc: 0.8717 - val_loss: 0.3627 - val_acc: 0.8386\n",
            "Epoch 132/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.3046 - acc: 0.8724 - val_loss: 0.3624 - val_acc: 0.8372\n",
            "Epoch 133/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.3008 - acc: 0.8747 - val_loss: 0.3632 - val_acc: 0.8370\n",
            "Epoch 134/500\n",
            "20109/20109 [==============================] - 13s 631us/sample - loss: 0.3025 - acc: 0.8744 - val_loss: 0.3621 - val_acc: 0.8386\n",
            "Epoch 135/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.3000 - acc: 0.8754 - val_loss: 0.3624 - val_acc: 0.8390\n",
            "Epoch 136/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.3017 - acc: 0.8736 - val_loss: 0.3612 - val_acc: 0.8396\n",
            "Epoch 137/500\n",
            "20109/20109 [==============================] - 13s 645us/sample - loss: 0.2991 - acc: 0.8765 - val_loss: 0.3611 - val_acc: 0.8396\n",
            "Epoch 138/500\n",
            "20109/20109 [==============================] - 13s 623us/sample - loss: 0.3000 - acc: 0.8745 - val_loss: 0.3617 - val_acc: 0.8384\n",
            "Epoch 139/500\n",
            "20109/20109 [==============================] - 13s 626us/sample - loss: 0.2963 - acc: 0.8762 - val_loss: 0.3615 - val_acc: 0.8400\n",
            "Epoch 140/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.2986 - acc: 0.8749 - val_loss: 0.3614 - val_acc: 0.8400\n",
            "Epoch 141/500\n",
            "20109/20109 [==============================] - 13s 624us/sample - loss: 0.2973 - acc: 0.8751 - val_loss: 0.3604 - val_acc: 0.8402\n",
            "Epoch 142/500\n",
            "20109/20109 [==============================] - 13s 622us/sample - loss: 0.2952 - acc: 0.8778 - val_loss: 0.3607 - val_acc: 0.8416\n",
            "Epoch 143/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.2960 - acc: 0.8762 - val_loss: 0.3601 - val_acc: 0.8410\n",
            "Epoch 144/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.2944 - acc: 0.8798 - val_loss: 0.3610 - val_acc: 0.8408\n",
            "Epoch 145/500\n",
            "20109/20109 [==============================] - 13s 648us/sample - loss: 0.2958 - acc: 0.8782 - val_loss: 0.3605 - val_acc: 0.8402\n",
            "Epoch 146/500\n",
            "20109/20109 [==============================] - 13s 642us/sample - loss: 0.2962 - acc: 0.8769 - val_loss: 0.3599 - val_acc: 0.8416\n",
            "Epoch 147/500\n",
            "20109/20109 [==============================] - 13s 641us/sample - loss: 0.2933 - acc: 0.8792 - val_loss: 0.3598 - val_acc: 0.8408\n",
            "Epoch 148/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.2925 - acc: 0.8800 - val_loss: 0.3601 - val_acc: 0.8410\n",
            "Epoch 149/500\n",
            "20109/20109 [==============================] - 13s 638us/sample - loss: 0.2920 - acc: 0.8802 - val_loss: 0.3603 - val_acc: 0.8422\n",
            "Epoch 150/500\n",
            "20109/20109 [==============================] - 13s 649us/sample - loss: 0.2914 - acc: 0.8798 - val_loss: 0.3592 - val_acc: 0.8428\n",
            "Epoch 151/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.2932 - acc: 0.8809 - val_loss: 0.3594 - val_acc: 0.8412\n",
            "Epoch 152/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.2885 - acc: 0.8817 - val_loss: 0.3589 - val_acc: 0.8436\n",
            "Epoch 153/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.2888 - acc: 0.8806 - val_loss: 0.3593 - val_acc: 0.8447\n",
            "Epoch 154/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.2877 - acc: 0.8839 - val_loss: 0.3601 - val_acc: 0.8422\n",
            "Epoch 155/500\n",
            "20109/20109 [==============================] - 13s 629us/sample - loss: 0.2852 - acc: 0.8819 - val_loss: 0.3589 - val_acc: 0.8457\n",
            "Epoch 156/500\n",
            "20109/20109 [==============================] - 13s 632us/sample - loss: 0.2880 - acc: 0.8810 - val_loss: 0.3594 - val_acc: 0.8445\n",
            "Epoch 157/500\n",
            "20109/20109 [==============================] - 13s 640us/sample - loss: 0.2875 - acc: 0.8814 - val_loss: 0.3585 - val_acc: 0.8441\n",
            "Epoch 158/500\n",
            "20109/20109 [==============================] - 13s 643us/sample - loss: 0.2882 - acc: 0.8825 - val_loss: 0.3591 - val_acc: 0.8449\n",
            "Epoch 159/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.2870 - acc: 0.8822 - val_loss: 0.3584 - val_acc: 0.8445\n",
            "Epoch 160/500\n",
            "20109/20109 [==============================] - 13s 630us/sample - loss: 0.2881 - acc: 0.8811 - val_loss: 0.3582 - val_acc: 0.8453\n",
            "Epoch 161/500\n",
            "20109/20109 [==============================] - 13s 627us/sample - loss: 0.2874 - acc: 0.8817 - val_loss: 0.3579 - val_acc: 0.8447\n",
            "Epoch 162/500\n",
            "20109/20109 [==============================] - 13s 633us/sample - loss: 0.2840 - acc: 0.8847 - val_loss: 0.3578 - val_acc: 0.8455\n",
            "Epoch 163/500\n",
            "20109/20109 [==============================] - 13s 637us/sample - loss: 0.2839 - acc: 0.8839 - val_loss: 0.3583 - val_acc: 0.8453\n",
            "Epoch 164/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.2864 - acc: 0.8800 - val_loss: 0.3573 - val_acc: 0.8447\n",
            "Epoch 165/500\n",
            "20109/20109 [==============================] - 13s 634us/sample - loss: 0.2843 - acc: 0.8846 - val_loss: 0.3582 - val_acc: 0.8451\n",
            "Epoch 166/500\n",
            "20109/20109 [==============================] - 13s 624us/sample - loss: 0.2832 - acc: 0.8847 - val_loss: 0.3574 - val_acc: 0.8465\n",
            "Epoch 167/500\n",
            "20109/20109 [==============================] - 13s 636us/sample - loss: 0.2835 - acc: 0.8842 - val_loss: 0.3577 - val_acc: 0.8457\n",
            "Epoch 168/500\n",
            "20109/20109 [==============================] - 13s 648us/sample - loss: 0.2816 - acc: 0.8820 - val_loss: 0.3574 - val_acc: 0.8451\n",
            "Epoch 169/500\n",
            "20109/20109 [==============================] - 13s 623us/sample - loss: 0.2784 - acc: 0.8848 - val_loss: 0.3576 - val_acc: 0.8457\n",
            "Epoch 170/500\n",
            "20109/20109 [==============================] - 13s 628us/sample - loss: 0.2815 - acc: 0.8834 - val_loss: 0.3584 - val_acc: 0.8447\n",
            "Epoch 171/500\n",
            "20109/20109 [==============================] - 13s 639us/sample - loss: 0.2813 - acc: 0.8852 - val_loss: 0.3577 - val_acc: 0.8447\n",
            "Epoch 00171: early stopping\n",
            "--- 2265.2733216285706 seconds ---\n",
            "INFO:tensorflow:Copying TPU weights to the CPU\n",
            "20109/20109 [==============================] - 159s 8ms/sample - loss: 0.2623 - acc: 0.8924\n",
            "5027/5027 [==============================] - 39s 8ms/sample - loss: 0.3575 - acc: 0.8448\n",
            "Train Accuracy: 0.8924\n",
            "Train Loss: 0.2623\n",
            "Validation Accuracy:  0.8448\n",
            "Validation Loss: 0.3575\n",
            "5027/5027 [==============================] - 15s 3ms/sample\n",
            "[[1003  497]\n",
            " [ 283 3244]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.78      0.67      0.72      1500\n",
            "         1.0       0.87      0.92      0.89      3527\n",
            "\n",
            "    accuracy                           0.84      5027\n",
            "   macro avg       0.82      0.79      0.81      5027\n",
            "weighted avg       0.84      0.84      0.84      5027\n",
            "\n",
            "RMSE: 0.3939\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lhZW6ObDXI8i",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "e54b2964-c7ca-4756-984f-5cbb18eccd27"
      },
      "source": [
        "print(\"Average Train Accuracy: %.4f%% (+/- %.4f%%)\" % (np.mean(tr_acc_array), np.std(tr_acc_array)))\n",
        "print(\"Average Train Loss: %.4f%% (+/- %.4f%%)\" % (np.mean(tr_loss_array), np.std(tr_loss_array)))\n",
        "print(\"Average Validation Accuracy: %.4f%% (+/- %.4f%%)\" % (np.mean(te_acc_array), np.std(te_acc_array)))\n",
        "print(\"Average Validation Loss: %.4f%% (+/- %.4f%%)\" % (np.mean(te_loss_array), np.std(te_loss_array)))\n",
        "print(\"Average Time: %.4f%% (+/- %.4f%%)\" % (np.mean(time_array), np.std(time_array)))\n",
        "print(\"Average RMSE: %.4f%% (+/- %.4f%%)\" % (np.mean(rmse_array), np.std(rmse_array)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Average Train Accuracy: 0.8905% (+/- 0.0033%)\n",
            "Average Train Loss: 0.2676% (+/- 0.0066%)\n",
            "Average Validation Accuracy: 0.8415% (+/- 0.0032%)\n",
            "Average Validation Loss: 0.3696% (+/- 0.0073%)\n",
            "Average Time: 2079.7712% (+/- 167.3976%)\n",
            "Average RMSE: 0.3981% (+/- 0.0040%)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eaXCaRhZ7Otf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dataset = pd.DataFrame({'Train Acc': tr_acc_array, 'Train Loss': tr_loss_array, 'Validation Acc': te_acc_array, 'Validation Loss': te_loss_array, 'Time': time_array, 'RMSE': rmse_array})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z4b7Ae3vSO7A",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "a943c4ca-3207-4e9e-fbb4-dc48dda748b6"
      },
      "source": [
        "dataset"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Train Acc</th>\n",
              "      <th>Train Loss</th>\n",
              "      <th>Validation Acc</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Time</th>\n",
              "      <th>RMSE</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.884971</td>\n",
              "      <td>0.278780</td>\n",
              "      <td>0.842482</td>\n",
              "      <td>0.369706</td>\n",
              "      <td>1837.398070</td>\n",
              "      <td>0.396885</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.890596</td>\n",
              "      <td>0.265821</td>\n",
              "      <td>0.838472</td>\n",
              "      <td>0.370386</td>\n",
              "      <td>2093.989715</td>\n",
              "      <td>0.401905</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.889552</td>\n",
              "      <td>0.270859</td>\n",
              "      <td>0.837080</td>\n",
              "      <td>0.380408</td>\n",
              "      <td>1949.700225</td>\n",
              "      <td>0.403634</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.894972</td>\n",
              "      <td>0.260387</td>\n",
              "      <td>0.844639</td>\n",
              "      <td>0.369960</td>\n",
              "      <td>2252.494309</td>\n",
              "      <td>0.394159</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.892386</td>\n",
              "      <td>0.262343</td>\n",
              "      <td>0.844838</td>\n",
              "      <td>0.357546</td>\n",
              "      <td>2265.273781</td>\n",
              "      <td>0.393906</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Train Acc  Train Loss  ...         Time      RMSE\n",
              "0   0.884971    0.278780  ...  1837.398070  0.396885\n",
              "1   0.890596    0.265821  ...  2093.989715  0.401905\n",
              "2   0.889552    0.270859  ...  1949.700225  0.403634\n",
              "3   0.894972    0.260387  ...  2252.494309  0.394159\n",
              "4   0.892386    0.262343  ...  2265.273781  0.393906\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": false,
        "id": "3MQQcmILVxv3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# import time\n",
        "# from keras.callbacks import ReduceLROnPlateau, EarlyStopping\n",
        "# start_time = time.time()\n",
        "\n",
        "# es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=7)\n",
        "\n",
        "  \n",
        "# history = tpu_model.fit(x_train, y_train\n",
        "#                     ,epochs=epochs, verbose=1 \n",
        "#                     ,validation_split=0.2\n",
        "#                     ,batch_size=128 * 8\n",
        "#                     ,validation_data=(x_test, y_test)\n",
        "#                     ,callbacks=[es]\n",
        "#                    )\n",
        "# print(\"--- %s seconds ---\" % (time.time() - start_time))\n",
        "# # tpu_model.save_weights('./tpu_model.h5', overwrite=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kB09B2pkVxv4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# loss, accuracy = tpu_model.evaluate(x_train, y_train, verbose=1)\n",
        "# print(\"Training Accuracy: {:.4f}\".format(accuracy))\n",
        "# loss, accuracy = tpu_model.evaluate(x_test, y_test, verbose=1)\n",
        "# print(\"Testing Accuracy:  {:.4f}\".format(accuracy))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HSoKam1eVxwA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "outputId": "3ec020e8-bc07-4336-eae9-7fbfc3208906"
      },
      "source": [
        "plot_history(history)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3EAAAFACAYAAAASzfVtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdeXhM1//A8fcs2Sb7QmIvYgmqSgiq\nCLEVFfu+1NqWUjv9UUqp1tJWafmiUakllFqKlqjWkn5DG9TWotRSIZGQPZPMzPn9ka+pWIOQxef1\nPJ7HvXOXz7kzk3M+c885V6OUUgghhBBCCCGEKBC0eR2AEEIIIYQQQoickyROCCGEEEIIIQoQSeKE\nEEIIIYQQogCRJE4IIYQQQgghChBJ4oQQQgghhBCiAJEkTgghhBBCCCEKEEnixGP5448/0Gg0/Prr\nr3kdyj19//33aDQarl279kTPs2jRIpycnB7qvOnp6Wg0Gr755pvHPn+3bt1o06bNYx9HCCFE/iJ1\n7b+elbo2N2MWhZMkcYWcRqO577/nnnvusY5foUIFoqOjqVGjRu4EXIg0adKE6OhoPD09c/W4S5cu\nxd7e/o71ixcv5uuvv87VcwkhhHgwqWvzjtS14lmlz+sAxJMVHR1t/X9ERAQdO3YkKiqKYsWKAaDT\n6e66X0ZGBra2tg88vk6nw8fHJ3eCLWRsbW2f6rVxdXV9aufKj8xmMxqNBq1WfpsSQjxdUtfmHalr\nxbNKWjuFnI+Pj/Wfh4cHAEWKFLGuK1KkiHW79957j8GDB+Ph4UGzZs0AmDNnDtWrV8fR0ZHixYvT\nq1cvYmJirMe/vYvHzeUNGzbQqlUrDAYDvr6+rFq16r5xxsbG0r17d0qVKoWDgwOVK1fms88+y7bN\nzS4MCxcupHTp0ri6utKhQ4c7ulDMnTuX4sWLYzAYaN26NZcvX77vuT/77DO8vLzIzMzMtv69996j\nQoUKAGRmZjJgwADKlSuHg4MD5cuXZ8qUKXfsc6u7dfHYsWMHVatWxd7enhdffJF9+/bdsd/YsWOp\nXLkyBoOB0qVL89Zbb5GcnGw95qBBgzAajdZfeF9//fVs1+cmpRQffPABzz33HLa2tvj6+rJw4cJs\n5/Lx8WHGjBkMHToUNzc3fHx8GD9+PBaL5b7X7H4x3hQZGUmzZs1wdnbG2dmZunXrEhUVZX19+/bt\n1K9fH4PBgJubG4GBgVy4cOGuZYE7fxWdMGEC1apV4+uvv6ZixYrY2dlx/vx5IiMjad68OUWKFMHZ\n2ZmAgAB27dqV7VgZGRlMnjyZsmXLYmtrS8mSJRkzZoz13K+++uodZa5fvz5Dhw6973URQjybpK6V\nuvZJ1LW3u3TpEp07d8bV1RWDwUDTpk05cuSI9XWj0cjw4cMpUaIEdnZ2FC9enL59+1pfP3LkCEFB\nQbi6uuLo6EiVKlUICwt7qBhEPqLEM2P37t0KUBcvXrzjNW9vb+Xs7KxmzJihTp06pU6ePKmUUmrO\nnDlq165d6uzZs2rfvn2qdu3aqnnz5tb9Tp48qQB18ODBbMu+vr5q/fr16vTp02r06NHKxsZGnTt3\n7p6xnT9/Xn300UcqKipKnT17VoWEhCh7e3u1atUq6zZdu3ZVrq6uqk+fPurYsWNq7969qmTJkmrg\nwIHWbdasWaP0er2aP3+++vPPP9WiRYuUl5eXAlRsbOxdzx0bG6tsbGzUxo0bs60vV66ceu+995RS\nSqWlpanJkyeryMhIde7cObVhwwbl5eWlZs6cad3+iy++UI6Ojtbl7du3Zzvv33//rezs7NTgwYPV\niRMn1Pbt25Wfn58C1Lp166z7TZ06Ve3du1edO3dO/fDDD6p8+fJq8ODBSimljEajmjt3rrKzs1PR\n0dEqOjpaJSQkWK9P69atrceZM2eOMhgM6ssvv1SnTp1Sn332mbKxsVFff/11tvfd3d1dzZkzR506\ndUqtXLlSabXabNvczf1iVEqpqKgoZW9vr3r37q1+/fVX9eeff6qvv/5aHThwQCml1NatW5VWq1Vj\nxoxRR44cUcePH1eLFy9WZ86cuWtZlFJqyZIlys7Ozro8fvx4ZTAYVJMmTdSBAwfUyZMnVXJystq5\nc6dasWKFOn78uPrjjz/U2LFjlZ2dnTp79qx13y5duihvb2+1atUqdebMGRUREaE+/fRTpZRSP/74\no9LpdOrSpUvW7Y8dO6YAdfjw4fteFyGEkLpW6trcqGvT0tKyxWw2m9ULL7ygatWqpSIiItSRI0dU\ncHCw8vLyUtevX1dKKTVjxgz13HPPqZ9//lmdP39eRUZGqvnz51uPWaFCBdW3b1914sQJ9ddff6nv\nvvtObdu27Z4xiPxNkrhnyIMqlldeeeWBx4iIiFCAunbtmlLq3hXLwoULrfsYjUZla2urli9f/lDx\nDh48WLVp08a63LVrV1W8eHGVkZFhXTd16lT13HPPWZdr1aql+vfvn+04Q4cOvW/FopRS7dq1Ux06\ndLAu7927V2k0mvtWhjNnzlTVqlWzLj+oYhk9erTy9fVVZrPZus26devuqFhut2rVKuXk5GRdvj2Z\nuen2isXLy0tNnjw52zavv/668vPzsy57e3urzp07Z9umcePGql+/fveMJycxdurUSfn7+yuLxXLX\n7f39/VXHjh3vebycJnE6nU5dvnz5gfFVrFhRzZkzRyn1b0K2ZcuW+24/ffp06/Lbb7+t6tSp88Dz\nCCGE1LVS1+ZGXXt7Evfdd98pjUajTp8+bd0mJSVFeXp6qg8//FAplfVetmzZ8q51r8ViUXZ2dmr1\n6tX3PKcoWKQ7pbCqU6fOHevCw8Np1qwZpUqVwtnZmaCgIADOnz9/32PdOvja1tYWLy8vrl69es/t\nTSYT77//PtWrV8fT0xMnJydCQkLuOE/VqlWxsbGxLhcvXjzbcU+ePEn9+vWz7dOgQYP7xgrQt29f\ntm7dyvXr1wFYsWIFL7/8crbB6J9//jm1a9emaNGiODk58d577z3wOtzqxIkT1K1bN9uYrbvFFhYW\nRoMGDShWrBhOTk7079+f5ORk4uPjc3yumJgYrl27RsOGDbOtb9SoEadPn87WNeX2gfK3X9O7eVCM\nv/32G82aNUOj0dyxr1KKQ4cO0bx58xyX515KlSplHXNy05UrVxgyZAiVKlXC1dUVJycnzpw5Y32v\nfvvtNzQajfWzfDeDBw9m2bJlKKUwGo2EhoYyaNCgx45XCCGkrpW6FnJW197q+PHjFC9eHF9fX+s6\ng8GAv78/x48fB2DgwIEcOHCAihUr8uabb/Ltt99aY9BoNIwZM4bevXvTpEkTpk2blq0rpih4JIkT\nVo6OjtmWz5w5Q5s2bahUqRJhYWH8+uuvrFu3DsgaU3Q/tw/U1mg09+37/cEHHzBv3jxGjx5NeHg4\nhw8fpk+fPnec52GPm1OtW7fGycmJsLAw0tPTWbduXbZ+5KGhoYwaNYrevXuzfft2Dh06xPjx4x94\nHR7Wnj176NGjB82aNWPTpk1ERUUxf/584MHX/FE97DV9GjFqtVqUUtnW3W1MxO2fWYCePXty4MAB\n5s6dy/79+zl8+DBVqlTJFtvNMQ730q9fP6Kjo9m5cyfffvstGRkZdO/e/TFKJIQQWaSulboWcu+a\n3qp27dr8/fffzJo1C61Wy9ChQ/H39yclJQWA999/n5MnT9KhQwcOHTpE7dq1mT59eq7GIJ4eSeLE\nPUVGRpKZmcknn3xC/fr1qVSpEleuXHki59qzZw9t27alb9++vPjii/j6+nLq1KmHPo6fnx8RERHZ\n1u3fv/+B+9na2tKtWzdCQ0PZvHkzRqORzp07Z4svICCA4cOHU6tWLSpUqMC5c+ceKrYqVaoQGRmZ\n7Y/27bHt3buXkiVLMmXKFOrUqUPFihW5ePHiHbGazeb7nqto0aJ4eXmxZ8+ebOt//vlnKlasmO0X\n1oeVkxhr1arFzp0770jEIKvievHFF9mxY8d94799kPytk6Lci1KKvXv3Mnz4cNq0aUO1atUoUqRI\ntl9xa9WqhcViYefOnfc8jqenJ506dWLJkiUsWbKE7t273zVhFEKIxyV1rdS1OVG1alUuX77MmTNn\nrOtSU1P59ddfqVatmnWds7MzHTt2ZMGCBURERPD7779ne698fX0ZNmwY3377Le+88w6LFi3KtRjF\n0yVJnLinihUrYrFY+Pjjjzl37hzr16/ngw8+eCLnqlSpEuHh4ezdu5c///yTcePGPdJt/tGjRxMa\nGsrChQs5ffo0S5YsyfHMS3369CEiIoKZM2fSvn17nJ2ds8UXFRXF1q1bOXPmDHPmzOG77757qNiG\nDRvG+fPnGTp0KCdPnmTHjh1MmTIl2zaVKlXin3/+ITQ0lLNnz/Lll1+ydOnSbNuULVsWk8nEtm3b\nuHbtmvUXtttNnDiRuXPnEhISwunTp1mwYAHLli3jnXfeeai4b5eTGCdMmMDvv/9O3759+e233zhz\n5gxr1qzh4MGDALz77rts2LCBsWPHcvToUf744w+WLVvGX3/9BUBQUBCHDx9myZIl/PXXX3zxxRds\n3LjxgbFpNBoqVqxIaGgox48fJyoqim7dumXbpmrVqnTs2JFBgwaxevVqzp49y4EDB1iwYEG27YYM\nGcLGjRvZvXs3gwcPfpxLJoQQ9yR1rdS1OdGqVSuqV69O9+7d+eWXXzh69Cg9e/ZEo9FY66gPPviA\n1atXc+LECc6ePUtISAg2Njb4+voSHx/P8OHD2b17N3///Te//fYbO3fupEqVKrkap3iK8nREnniq\nHjTYevbs2XesnzdvnipRooSyt7dXjRo1Ulu2bFGA+uWXX5RS9x5sfXP5phIlSqgPPvjgnrFdu3ZN\ntW/fXjk5OSlPT081YsQINW7cOFWpUiXrNjmZ7EIppT766CPl4+Oj7O3tVYsWLdTSpUsfONj6psqV\nKytAff/999nWp6enq9dee025ubkpFxcX1bt3b+vMVTc9aLD1zXV+fn7K1tZWVa9eXe3YsSPbwGWL\nxaLGjRunvLy8lMFgUG3btlUrVqxQgIqOjrYe54033rDOBDZkyJC7Xh+LxaJmzJihypQpo2xsbFT5\n8uXVggULspXrbu97z549VYsWLe55jXIa4/79+1VgYKAyGAzKyclJ1atXT0VFRVlf37Jli6pdu7ay\ns7NTrq6uqkmTJur8+fPW1999911VrFgx5eTkpHr37q3mzZt3x8QmVatWvSO+qKgoVadOHWVvb6/K\nli2rlixZol566SXrdVIq6/2cMGGCKlWqlLKxsVElS5ZUY8eOveNYlStXVjVr1rzntRBCiNtJXSt1\nbW7UtbdPbKKUUhcvXlQdO3ZULi4uysHBQQUGBqpDhw5ZX58/f76qUaOGcnJyUk5OTqpOnTpq69at\nSimlkpKSVNeuXVWZMmWUra2tKlq0qOrRo0eOJgcT+ZNGqbv0dxJCiGec0WikVKlSTJ8+nSFDhuR1\nOEIIIYQQVvq8DkAIIfITs9lMXFwc8+fPx2Kx0Lt377wOSQghhBAiG0nihBDiFqdPn8bPz48SJUqw\nfPlyDAZDXockhBBCCJGNdKcUQgghhBBCiAJEZqcUQgghhBBCiAJEkjghhBBCCCGEKEAkiRNCCCGE\nEEKIAiRfTmxy+fLlR97Xy8uLa9eu5WI0eaewlKWwlAOkLPmVlCV/yklZihcv/pSiKTwetY581j5b\nBYWUJX+SsuQ/haUckDv1o9yJE0IIIYQQQogCJF/eiRNCCCEKusOHDxMSEoLFYqFp06YEBwdne335\n8uUcP34cgIyMDBISEli+fHkeRCqEEKKgkSROCCGEyGUWi4Vly5YxadIkPD09mThxIv7+/pQsWdK6\nTb9+/az/3759O+fOncuDSIUQQhREBSKJU0qRnp6OxWJBo9Hcd9urV69iNBqfUmRPVkEqi1IKrVaL\nvb39A98jIYQo7M6cOYOPjw/e3t4A1K9fn4MHD2ZL4m61f/9+unTp8jRDFEIUYre2nc1mc4FpT95P\nQWoXP8jNsjxO+7lAJHHp6enY2Nig1z84XL1ej06newpRPXkFrSwmk4n09HQcHBzyOhQhhMhT8fHx\neHp6Wpc9PT05ffr0XbeNjY0lJiaGatWqPa3whBCF3K1t54LWnryXwlIOyF6WR20/F4gkzmKx5CiB\nE3lLr9cXml9IhBDiadm/fz9169ZFq737XGPh4eGEh4cDMGvWLLy8vB7pPHq9/pH3zW+kLPmTlCX/\nuHr1KnZ2dtblwtKOLizlgH/Lotfr0Wg0D/15KxBXQrrnFRzyXgkhBHh4eBAXF2ddjouLw8PD467b\nRkREMGDAgHseKygoiKCgIOvyo06x/axNz11QSFnyp4JeFqPRaL3To9frMZlMeRzR4yss5YA7y2I0\nGu/4vD3oEQMFIonLa/Hx8XTt2hXI6vai0+mslfHWrVuxtbV94DFGjhzJ0KFD8fX1faKxCiGEyHvl\ny5cnOjqamJgYPDw8iIiIYPjw4Xds988//5CSkkLFihXzIEohhMh9edFuXrVqFX/88QfTpk179MAL\nGEnicsDDw4OdO3cCMHfuXBwdHXn99dezbaOUsg5OvJuPP/74iccphBAif9DpdPTv358ZM2ZgsVgI\nDAykVKlShIWFUb58efz9/YGsrpT169eXXgxCiEJD2s1Phzzs+zGcO3eOxo0bM2zYMAIDA7l69Srj\nxo2jVatWBAYGZvsABgcHc+zYMUwmE35+fsycOZOgoCDatm1719v1v/32G6+88grNmzenXbt2nD17\nFsga/DhlyhSaNGlCUFCQ9ZlCUVFRtG3blqCgINq0aUNaWtpTuQZCiGePxQKLFjmSlCSJx/3UrFmT\nTz/9lM8++4wOHToA0LVrV2sCB9ClSxd69uz51GKKjdWyfLnhqZ1PCCFuetx2c2Bg4D3bzbe6cOEC\nnTp1IigoiG7dunH58mUANm3aZG0/d+rUCYCTJ0/yyiuv0KxZM4KCgjh//vyTuwC5TJK4x3TmzBkG\nDRrETz/9RLFixZg4cSLbt29n586d7Nmzh1OnTt2xT2JiInXr1iU8PJxatWqxZs2aO7apUKECmzdv\nZseOHbz99tt89NFHAKxYsYKrV6+yc+dOwsPDadeuHenp6bz55pvMnDmT8PBwVq1alaNb1UKIZ0t4\nuB3t23vy5ZeO3LjxbwL2xx96Xn3Vi6CgIjRrVoSxY13ZvNkeiyXr9agoG2rV8mbhQicyMmDkSDem\nT3dl40aZibagWbMMIv/vZ7780jGvQxFCPIMep928e/fue7abb/XOO+/Qo0cPwsPDadOmDVOmTAFg\n3rx5hIWFER4ezrJlywD46quvGDJkCDt37mTr1q3Wx8IUBAWuO+W777pw4oTNPV/XaDQopR7qmFWq\nZDJtWuIjxVOmTBleeOEF6/KmTZtYvXo1ZrOZK1eucOrUqTvGOtjb29OkSRMAqlevTmRk5B3HTUxM\n5O233+bvv//Otn7v3r0MHDjQOljV3d2dY8eOUaJECZ5//nkAXFxcHqksQoj8bcQIN2JitCxefB0X\nl4f7Oxcbq+Xtt90wGjUcOGDH3LnO7NoVg4+PhQULnDh5Uk+jRkbS0zVs3erAqlWOHDiQzIQJSbz1\nljvXr2uZOdOFZcscuXpVx5gxifTqlfqESiqelImWGTixkLaTt+DtXY/WrdPzOiQhxBM2aZITx47l\n7tT8j9p2flLt5lsdOnSIr776CoBOnToxe/ZsAGrXrs2IESNo06YNrVq1AsDf35/58+fzzz//0KpV\nK8qWLfvQZcorcifuMRkM/3ZLOXv2LEuXLmXt2rWEh4cTGBh41yn3b71LptPpMJvNd2zz4YcfEhgY\nyI8//siyZctk6n4hnnE7dtjxzTcG9uyxp1s3T65fz96V0WyG9esdmDTJhSVLHPn1139/7FIKJk50\nJTVVy9at19i4MZbkZA2ffebMlStatmxxoGfPVJYuvc7XX8dz9OgVBg9OJiTEiVatinDhgo7Vq+OY\nNesGRqOGadMSGDkyGRnGVfCkvD2CTL8qhOl6MH9oDCdOFLjfcoUQBdiTajfnxOzZsxk9ejQXL16k\nZcuW3Lhxg06dOrF06VJsbW3p1asX//3vfx/p2HmhwP31flDWn5fTjyYnJ+Pk5ISzszNXr17lp59+\nonHjxo90rMTERHx8fABYu3atdX3Dhg0JDQ2lbt266HQ6rl+/ToUKFfjnn384evQozz//PElJSRgM\nhkLzQEQhnjUWC+zda8eqVQbc3Cy8/noykya5UrlyJuPGJfHGG+40a1aUgQOT8ffP4PhxG0JDHTl5\n0gZ7e0V6elZ21bhxOu3apfHTT3Zs3+7ApEkJVKyY9fexa9dUVq40kJqqwWyGfv1SrOfX6WDy5ETi\n47V8842Bt95KIiAgg4CADHr1SpXkrQBTBgM3vgrBs0Urvk1sR8fXI9nwgxkHh4e7syuEKDjefz85\nX07Nn5vt5lvVrFmTLVu2EBwczIYNGwgICADg/Pnz1KpVi5o1a7Jr1y6uXLlCQkICZcuWZeDAgVy4\ncIGTJ09St27dx47haShwSVx+9vzzz1OhQgUaNmxIyZIlqV279iMfa+jQoYwePZo5c+YQGBhoXd+r\nVy/OnTtHUFAQOp2OPn360KdPHxYuXMjEiRNJT0/H3t6edevWPfST34UQT1dysoYLF3TExurw8LBQ\nsWImGzdqePfdIvz5pw0eHmaSk7WsXGlAKQ3ffnuNOnUyCAuL48MPnZk+3dV6rOeeM/H55/G0bZv+\nv+TLgc8+c+ann+xxc7PQt28Kgwf/m6iNGJHEunUG1q410KxZOs89l/2XTa0W5s69QadOqdSrl2Fd\nLwlcwWcuUYIbi7/At0sXev41k6lTp/Dhhwl5HZYQ4hmTm+3mW82YMYNRo0axYMECvLy8mDdvHgBT\np07l4sWLKKVo2LAhlStX5pNPPmHTpk3o9Xp8fHwYPXp0rsTwNGjUww4gewpuziJzU2pqarbbr/dT\nmB8EWBDc7b0q6A/MvJWUJX/Kj2W5cUPDa6950LJlOoMHp2RLfk6e1LNkiRObNjlY75oBaLUKi0WD\nr28mb72VTNu2aVy5omP2bGfKlDEzdmxStnMcO6bn4kU91aplUrKk+Y4EKyFBw99/66laNRP9XX6y\nmzTJhZAQJ1avjqNhw9zvsp2T9+VBDzMVd7q9jsyp298P17FjsV8dRi31K1M2lCQgIOM+e+cv+fE7\n/6ikLPlTQS/Lre2xgtievJvCUg64syx3az/Lw76FECIPfPqpMwcO2HHggB3Hjtnw0Uc3cHCAX36x\npUcPT3Q6RadOqbz0khFvbwtXr2o5ftyGmjUdaNo01pp0lSljZsGCG3c9R7VqJqpVu3eF5uqqeOGF\nzHu+PmFCVjfJl1+WMbfPosT/+z/sfthBSOIgBk/fw8YtGXKnVQghCghJ4oQQIhdcvKhj6FB3undP\npV49IyEhjnTrlkLJkmbmzHHh8GFb3norialTXSld2sSGDXF4elqyHePVV9Px8rLjaf346+SkaNtW\nZid8Vik3N5LencyLI0ZQ+tA2vv8+kFat5PMghBAFgcxOKYQQ95GQoGH5cgNpaVm3KJSCs2d13NoR\nPTMT3nzTnd9+s2XMGDc6dvTCxkYxblwSI0cms3r1NTIzYeRId2xtFV9/HX9HAidEXkhr357MsmV5\nz24GH8x0opD0VBJCiEJPkjghhLjFhQs61q51wGTKStjGjXPj//7PjT59PEhI0DB2rCsvv+xNjx4e\nXLyoIzMTPvrImagoWxYuvM6wYUlcuaJj6NBkvL2zErWGDTP48cdYJk9OYM2aOEqVerTpkYXIdTod\nKUOHUs14iHJnf+KHH+zzOiIhhBA5IN0phRDPtKNHbQgNNVCsmJn4eC2hoY5kZmrYudOeoKB0vvvO\ngaCgdH780Y46dbxJTtbSrl0q4eH2vPRSUczmrDt0PXumEBycRnAw9OmTSvHi2RM1g0Hx+uspdwtB\niDyV2rEjTnPmMDV+BhNCd8gDwIUQogCQJE4I8cw6e1ZH9+4epKVpMRqzps/v1i2VEiXMzJ7twrZt\nDtSsmcGyZfF8950D06e78MEH1+nQIY1//tGyYoUjdnaKkiXNtGuXZj1uiRJyp00UILa2pAwZQv33\n3iNx70nOni1GuXLyGRZCiPxMulPmQKdOnfjpp5+yrVuyZAkTJky4734VKlQA4MqVKwwaNOiexz5y\n5Mh9j7NkyRLS0v5tIPbu3ZuEBHmmjxB3k5io4fLlrD9tGRkwYYIrzz/vTYMGRXntNXd+/90GgL/+\n0tG7tycaDYSHx3D6dDRHjlxh9uwE3n47mXnzrlO5ciaffHIdvR6Cg9P47berdOiQ9V0sUcLCxIlJ\njBqVTJcuadjZ5VmRhXhsqZ06YdHb0EcTSmioY16HI4Qo4Apr23nu3LksWrTosY+TGySJy4Hg4GA2\nbdqUbd2mTZsIDg7O0f4+Pj4sWbLkkc+/dOnSbB/E0NBQXF1d77OHEM+mf/7R0qpVEerW9WbMGFd6\n9/YkNNSR+vUzeP75TA4etKVVqyK89FJRGjb05soVLcuXx1O2rBkHB/Dw+He2kq5d09i1K5by5eWO\nhCj8lIcHxqZN6Gu7mm/C7LilyhFCiIcmbecnT5K4HGjdujW7du0iIyPrQagXL17k6tWrBAQEkJKS\nQpcuXWjRogVNmzblhx9+uGP/ixcv0qRJEwDS0tJ44403aNSoEQMGDCA9/d+xBxMmTKBVq1YEBgYy\nZ84cAJYtW8bVq1fp3LkznTp1AiAgIID4+HgAFi9eTJMmTWjSpIn1w37x4kUaNWrE2LFjCQwMpHv3\n7tk+yDft2LGDNm3a0Lx5c7p27UpsbCwAKSkpjBw5kqZNmxIUFMTWrVsB2L17Ny1atCAoKIguXbrk\nyrUV4lHduKHh++/tmTnTmfHjdYSFOdC5sxdxcVq6dk1l/XoD//2vLR9/fJ3Fi6/zxRfXiYiI4a23\nkihTxsSUKQns2RNDrVr3fo6aEM+StPbt8TRGUyNhDz//LBOcCCEe3ZNoOzdo0CDP2863OnbsGG3a\ntCEoKIgBAwZw48YN6/kbN25MUFAQb7zxBgC//PILzZo1o1mzZjRv3pzk5ORHvrY3yZi4HHB3d6dG\njRrWJGbTpk20bdsWjUaDnYu9GJ8AACAASURBVJ0dy5Ytw9nZmfj4eNq2bUvz5s3R3OOJqStWrMDB\nwYGff/6ZEydO0LJlS+tr48ePx93dHbPZTNeuXTl+/DgDBgzgP//5D+vWrcPDwyPbsX7//XfWrl3L\nd999h1KKNm3aUK9ePVxdXTl37hwLFy5k9uzZDBkyhG3bttGxY8ds+9epU4ctW7ag0WhYtWoVn3/+\nOVOmTOGTTz7B2dmZXbt2AXDjxg3i4uIYO3YsGzZsoHTp0ly/fj2Xr7IQORMbq2XxYie++spAaqoW\nGxuFTgfp6e64uFhYvTqOF1/MZOTIJJKTtVSs+O+c6S4uigkTkvIweiHyr/SgICzOzvRPD2XzD/Vo\n2VImOBFCPJon0Xbet28fv//++33bzidOnHiibedbvf3220yfPp169eoxe/Zs5s2bx7Rp01i4cCG/\n/PILdnZ21i6cixYtYubMmdSuXZuUlBTs7R//h7ICl8S5vPsuNidO3PN1jUaDuvUBTjmQWaUKidOm\n3Xebm7eFb34Q586dC4BSilmzZhEZGYlGo+HKlSvExsZStGjRux4nMjKS/v37A1ClShX8/Pysr23Z\nsoWVK1diNpu5evUqp06dolKlSveM6cCBA7Rs2RKDwQBAq1atiIyMpHnz5pQqVYpq1aoBUL16dS5e\nvHjH/tHR0bzxxhvExMSQkZFB6dKlAdi7dy+ff/65dTs3Nzd27NhB3bp1rdu4u7vf93oJ8TgSEzVE\nRNjh62vC1zcrCYuO1vLFF06sXOlIRga0a5dGnz6pVK+egY+PF7/8coMiRSx4eWVN61+8uAWQZ7EJ\nkWMODqS1bk37bzYwfudnmM2g0+V1UEKIx+U0aRK6Y8dy9Zj5te18+vRpqlSpcs+YHrftfFNiYiIJ\nCQnUq1cPgM6dOzNkyBAA/Pz8GDZsGC1btrQmnLVr1+a9996jffv2tGrVCldXV0yP+WDOApfE5ZUW\nLVowdepUjh49SlpaGtWrVwdgw4YNxMXFsX37dmxsbAgICMBoND708S9cuMDixYvZunUrbm5uvP32\n2490nJvsbpllQafTZbv1fNPkyZMZPHgwzZs3JyIignnz5j3y+YR4WJmZ8OWXjsyf70zp0iYGDEgh\nOVnDjh32RETYkZmpQadT9OuXQkaGhrAwAxYLdOyYxrBhSdlmz9Prwc9PnlIsxONKb9cOzzVreOH6\nHn79NYCAgIy8DkkIUUDlRdv5bu3dnMpJ2zknVqxYwX//+1927tzJ/Pnz2bVrF8OGDaNp06b8+OOP\nBAcHExYWRtmyZR85ViiASdyDsn69Xv/Yme3dODo6Ur9+fUaNGpVtUGZSUhJeXl7Y2Niwf/9+Ll26\ndN/jBAQEsHHjRho0aMAff/zByZMnrcdxcHDAxcWF2NhYdu/eTYMGDQBwcnIiOTn5jlvCAQEBjBw5\nkmHDhqGU4vvvv2f+/Pk5LlNiYiI+Pj4ArFu3zrq+YcOGLF++nGn/u9Y3btygVq1avPPOO1y4cMHa\nnVLuxokHOXzYhnHj3GjVKo3XX0/GwSFr/eXLWvr08eTkSRsaNDASHa1lxIisz1O5ciYGDkyhceN0\ntmxx4MsvHdHroWvXVIYNS5YHZQvxBBkDArA4OPCKcTs//NBIkjghCoHk999/Im3jB8nttnPjxo0f\n2Ha+eWfsSbWdb3JxccHV1ZXIyEgCAgJYv349devWxWKxcPnyZV566SXq1KnD5s2bSUlJ4fr16/j5\n+eHn58fhw4c5ffr0s5fE5aXg4GAGDBjAF198YV3XoUMH+vbtS9OmTalevTq+vr73PUafPn0YNWoU\njRo1okKFCtZfJapWrUq1atVo2LAhxYsXp3bt2tZ9evbsSc+ePfH29uabb76xrn/++efp3LkzrVu3\nBqB79+5Uq1btvrd/bzV69GiGDBmCq6srL730knW/ESNG8M4779CkSRO0Wi2jRo3ilVde4aOPPmLg\nwIFYLBa8vLxYs2ZNzi6cKPQSEjRs2uTAq6+m4eaW1Z15zx5bBgzwQK+HOXNcWLvWwOuvJ1O7dgYD\nBngQH69l2bJ4WrRIRyk4eNAWDw8LFSr8W9E0aJDBm28mY2+v8PaWrpFCPHF2dmTUr0+7iO+Z88Mn\nTJ6cyD2GqQghxAPlZtu5QYMG+Pr65mnb+VaffPIJEyZMID09ndKlSzNv3jzMZjNvvfUWSUlJKKXo\n378/rq6uzJ49m4iICLRaLRUrVqRp06YPfb7badTDDiB7Ci5fvpxtOTU11dp39UGe1J24vFAQy3K3\n98rLy4tr167lUUS5S8pyp0OHbHjjDXcuXtRTqpSJGTMS+PFHe77+2kCFCiZWrozj9Gk977/vwtGj\ntgC4uVlYuTKOGjVyZ2ZIeV/yp5yUpXjx4k8pmsLj9joyp3L62TKEhOA2aRK+nGbZTy7ZfljJL561\n70lBIWXJP25tjxXE9uTdFJZywJ1luVv7+UH1ozxiQAjxyKKibGjf3gulYO7c6ygFffp48vXXhv9N\n838Nb28LDRpksH37NbZujeWNN5JZt+5ariVwQojcZWzUCIAW/MCePfIUeyGEyI+kO6UQ4pF9/LEz\nzs4Wvv8+Fnd3RcuW6YSFGWjRIp3nnss+dk2jgRo1MiV5EyKfM5cti6lMGYJjvmfOvv4MGJCS1yEJ\nIYS4jdyJE0LcU3y8lh077LDcZTjasWN6fvzRnoEDU3B3z+qV7eamGDIk5Y4ETghRgGg0GBs35uXM\n3fwWAYWk95IQQhQqBSKJy4fD9sQ9yHuVv33xhSOdOnly9erdv/r//a8to0a5sWePHRERtjRrVoTX\nXvOkd28PLlzQsW6dA5MnuxAVZcOCBc44OVno109+pReisElv3Bh7UwrVkiP5/XebvA5HCPGQpD1W\nsDzK+1UgulNqtVpMJhN6fYEI95llMpnQagvE7wLPpIwM+PxzJ+LjdQQHe7F9uwU3t39fv3BBx4AB\nHty4oSUsLGtwbblyJsaOTWT+fGfq1fMGQKdTfPmlEwBDhybh6ioVhRCFTUadOiiNhoZqD/v21aRm\nTekGLURBIm3nguNR288F4p21t7cnPT0do9GI5gFzHdvZ2T3WQ7Lzk4JUFqUUWq0We3v7vA5F3MPu\n3fbEx+sYMyaRZcscadLEhhUr9FSrZiItTcOAAR4oBT/+GMMff+i5eFFPv34pODkpmjdPZ9s2Bxo1\nMuLnl8mqVQb27rVj8GC5CydEYaTc3DD5+dHq758Zu288w4cn53VIQoiHcGvb2d7evsC0J++nILWL\nH+RmWR6n/VwgkjiNRoPDzacEP0BBnxL2VoWpLCLvrVvnQJEiZt56K5m2bdPp1asInTp50b9/Chs2\nOHDpko6vvoqnUiUTlSplHwRTpYqJKlWSrMuDB6dIAidEIWesV4+af67iyEFIS4McVsNCiHzg1rZz\nYWlPFpZyQO6URfq+CfEMiI/XEB5uT/v2aej14Otr4qefMvHxMfPpp84UK2ZmxYp4mjYtHL9wCSEe\nX0ZAAHbmNKpmHOLXX23zOhwhhBC3KBB34oQQD0cpOHFCT3i4PVev6oiN1ZKZqaFTp1TrNiVLwpYt\n1/jnHx2VK8v0c0KI7DICAgAI1P7Evn1VefnljDyOSAghxE2SxAlRiBw8aMv69Q6Eh9sTHa0DwM3N\nQkKChjp1jFStmj1Zc3ZWksAJIe7K4uVFZoUKtI75meH7RwFJD9xHCCHE0yFJnBAFSGYmGI0aHB0V\nt87xc+aMnunTXQgPt8dgsNCokZExY9Jp0sRI0aIWzGZ4wJxAQghxh4yAAGqt3czRw1oSEjQyG60Q\nQuQTOUriDh8+TEhICBaLhaZNmxIcHJzt9WvXrrFw4UJSUlKwWCz06NGDmjVrEhMTw8iRIylevDgA\nFSpUYPDgwblfCiGeARcu6OjUyZN//tFja5s1Y+TMmQlERdnw5pvu6HTwzjuJ9O+fgoND9oaWTpdH\nQQshCrSMevVw//prqnOE//63LC1apOd1SEIIIchBEmexWFi2bBmTJk3C09OTiRMn4u/vT8mSJa3b\nrF+/nnr16tG8eXMuXbrEBx98QM2aNQHw8fFh9uzZT64EQjwDYmO1dO/uSUqKlvHjE7l6VceqVQYi\nImy5cUNL1aqZLF8ej4+PJa9DFUIUIsaXXkJpNLTVbWPfvpGSxAkhRD7xwNkpz5w5g4+PD97e3uj1\neurXr8/BgwezbaPRaEhNzZowITU1FXd39ycTrRDPoJiYrATu6lUtK1bEMXx4MjNmJLB1ayylS5tp\n2TKd9evjJIETQuQ6S5EiZNaoQReHLezbZ5fX4QghhPifB96Ji4+Px9PT07rs6enJ6dOns23TuXNn\n3n//fb7//nuMRiOTJ0+2vhYTE8O4ceNwcHCgW7du+Pn55WL4QhQ+58/rmDTJFQ8PCy1apDN9ugux\nsVq+/PI6tWplWrerUsXE1q2F43kpQoj8Kz0oiMqH5nAjKZ6rV7V4e8sPRkIIkddyZWKT/fv307hx\nY9q2bcupU6f47LPPmDt3Lu7u7nz++ec4Oztz9uxZZs+ezdy5czEYDNn2Dw8PJzw8HIBZs2bh5eX1\nyLHo9frH2j8/KSxlKSzlgNwtS3o67N2r4cQJDX//raFMGUWxYjBypA6TCSwW+OYbA0WKKMLDTfj7\nOwPOuXJukPclv5KyiPwmPSgIl9mzacV2fvghmD59Uh+8kxBCiCfqgUmch4cHcXFx1uW4uDg8PDyy\nbfPjjz/yzjvvAFCxYkUyMzNJSkrC1dUVGxsbAMqVK4e3tzfR0dGUL18+2/5BQUEEBQVZlx/nCeby\nNPf8p7CUA3KnLGlpGiZOdGXbNntSUrJ6NBsMFlJTs2YfKVfOxFdfxeHtbWHnTntq1cqgVCkzuX0J\n5X3Jn561styc+ErkX6aqVTH7+NA9ZTMT1/aQJE4IIfKBB46JK1++PNHR0cTExGAymYiIiMDf3z/b\nNl5eXhw7dgyAS5cukZmZiYuLC4mJiVgsWd0url69SnR0NN7e3k+gGEIUHB995My6dQbatUsjNDSO\no0ejOX36ClFRV1i9Oo6tW2MpV86Mo6MiODiNUqXMeR2yeFYohTYuLutp8ULcpNGQ3rQpjY07OXYI\nTp+WpxMJIURee+BfYp1OR//+/ZkxYwYWi4XAwEBKlSpFWFgY5cuXx9/fnz59+rB48WK2bt0KwJtv\nvolGo+HEiROsXbsWnU6HVqtl0KBBODk5PfFCCZFfRUXZsHSpI717pzBrVkK217y9LXh7G/MoMlGQ\n6C5eRJOQgKlatYfb0WRCd+kSFk9PlJNTtocHaq9dw3XsWBx27CDTz4+Unj1Jb9UKi48P2itXsPn9\ndzLq1UM5516XXlFwpAcF4blyJW20WwkLa8akSYl5HZIQQjzTNErlv59cL1++/Mj7PmtdkQqCwlIO\nuH9ZMjNh1y57Spc2UaGCif/1JLZKT4dWrYqQlKRl9+4YnJ3z9qv3rLwvBc3dyqL/4w+UnR3msmUh\nI4OiTZuiP3uW1A4dSBo3DnOpUpCWhsO2bejOn8dUpQoYjTiuXIn+7FmML7+MuUgRDOvXo7tyBQCL\nmxvpjRqR+cIL6P/6C/vvv0ebnExKr17YRkZi+7/eFaaSJdFfumTdJ+W11zCVLQtAhr8/5jJlHqos\nt5PulA/vUevIx/qeZGZStHFjLlxzpr7Dbxz4NRZ9Ht6QK+zf+YJKypI/FZayFJZyQO7Uj9InQohH\ndOaMjv/8x4lp0xKwt4c1awxMmOAGgJOThblzb9CmTdYzlZSCsWPdOHXKhtDQuDxP4ETBoI2Px3nW\nLAyrVmHx8CD2++9x2L49K4Fr1w6H777DsGEDZm9vNOnpaBOy3901lSxJxosvYv/DD2iSkjA2bkzS\nyJFokpOx+fNP7HbvxrBpExY3NzJq1iTx//4PU+XKAOj//BP78HBsDh0itXdvMqtUwXHFCpw//th6\n/Ovz55N2nyROFCI2NiSNGUO5YcMITP6GZcteYciQlLyOSgghnlmSxAnxiD791JkNGwzUrZtBhw5p\nbNzoQLlyJkaNSiIkxJEhQzyYMCGRli3T2bzZgQ0bDIwbl0iTJtJlUtxFRga2hw7Biy+CrS3auDi8\n2rZFd+kSqb164fDtt3gMGoT+778xvvwyNxYuJOnCBex37MDm999BqyW1Sxcya9ZEf+IEmowMMgIC\nQKsFkwlNSgrK1TX7OS0WtNeuYSlSJFvXSgBTpUokV6qUbZ2xSRO0V6+i+d9zQS1FijzRSyLyl7R2\n7XBcuJA55yfz3LROODsrevSQSU6EECIvSBInxCOIj9eydasDAKtWGahb10hkpC2jRyfRvn0arVql\nMWKEO7NmuTBrlgsAHTqkMnx4cl6GLZ4wTUoKysEhK3G6m/R0sLEBnc66Snv5Mi6zZ2O/bRva5GSU\nmxv28+bhuGgRuitXiFu/nozatTG+/DIegwejNBoSJk8GjQZzmTKkDBp0x2kya9XKvkKvvzOBA9Bq\nsRQt+lBltMjkVM8urZak8eMp0a8fa0qOpPPY+cTHa3nzzeR7fuSFEEI8GZLECfEI1q51wGjU0L59\nKt9+a2D+fGeU0tCuXRoA9vbwxRfX6d07hbg4LRoNtGiRfvvNDlFYGI04z52L0+efYylalPTAQEwV\nKmDx8ckaN1a8OI5ffonLzJlYPDxI7dYNc9Gi6P/+G0NoKBqLhdQOHTA2aIDbokV49O8PQPznn5NR\nuzYA6a1bkzB9OphMmKpWzcvSimeYMSiI5IED6bh0IV9V8qLPB1PZt8+OBQuu4+UlDwEXQoinRZI4\nIR6SxQKhoY7UqWNk0qRENm92IDTUkRdeyKBcuX8fB6DVQoMGGXkYqXgsSqG9cgWNxYJycMDyv+dj\naq5fxzE0NKtLodmM7to1bKKisDlzhtQOHdBkZOCwbRvaxH9n7zMXLYouJob0Jk3AYsHpk0/QKIXS\naEhv0YLEKVMwly4NgKl7dzJGj8ZUvjzp7dplCynlf8mdEHlGoyFxyhS08fH03vAeZVuaaLb7Pdq3\n92LNmjhKlJBHogghxNMgSZwQDyEzE/7zH0f+/lvPmDFJ+PhYaNo0nR07HKx34UQ+YbFARkbWbdFb\nmUxZ/25frxS6v/7C9sgRbA8exG7XLvS3zAKY0r07acHBuI0di/7CBZRen9Ud0dMTc/HixIWEYGze\n3HosTVISukuXsNu7F9vISIxBQaR27w4aDdqYGDCbs8aU3T7Fn8FA4vTpT+CCCJFLtFpuzJsHQIMN\nM/izyV/UPBhCcLAnixdfp2bNzDwOUAghCj9J4oTIoYgIW8aPt+HsWVvq1TPyyitZSduQISmcOmVD\ncLAkcU+b7swZXGbMQB8fj/PLL5PWqhWmqlXR/fMP7oMHo7t8mbjQUExVq+IQFobj119jc/IkSqsl\ncepUUnv0AI0GTWoqbqNH47B5MwAWgwFjw4Ykv/kmysEBmxMncPzqKxxXr8bs40Ps5s13jju7lUaD\ncnHBVKUKpipVSBkyJNvLDzsOTYh8x8aGG/PnYypbltJz53Ku3O+0T1hB27a1aN8+lSlTEilSRLpX\nCiHEkyJJnBA5sHKlgXfecaVsWQgJiaNZM6N1fFvduhns3x+TtwEWcprUVGyOH8duzx7sdu1Cm5iI\n2csL28OHUfb2ULUqTp9+ivPHH5NRowa6CxfQZGSgnJ3x6tCBzJo1sdu7l4xq1Ujp0web48dxGzcO\nh3XrMFWpgm1kJPpTp0gaNYq01q0x+fpmu0OWBqT27o3Dt9+S0q+fJGFCAGg0JI8aRWa1ariNH8/O\nhLqE+4+g27aptIvyYtWqOJ57TrpXCiHEkyBJnBB3YbFAWJiB9esduHhRx6VLeho3TmftWi2ZmfKI\ngHtSCpvDh8msXBkcsmbv1CQloTEas7oeurv/O5W9UthGROCwaRNpHTqQUbeu9TC6ixex27UL26go\nbI4eRX/mTNbYNK2WzJo1yaxWDW1MDKldu5I0Zgwefn7EnzqFw8aNGFatwly6NNc//RTl6Ihn797Y\nHjhAwtSppAwYkDVY0WLBMSQEw6pVOGzahLKzI37lSowNG96zaKYKFUgaN+6JXj4hCiJj8+bE+Pvj\nOm0azdbN5ZLnKsbEvk+HV3vSuXsG7u4W2rRJp2RJSeiEECK3aJRS+e6pw5dvGYfysJ61p7kXBAWl\nHKdO6dm2zR4bG9i5046DB+2oXDkTP79Mnn8+kwEDUvDxKRhlyYlcf1+UwmXKFJyWLcPi6kp6ixbo\n//wT2yNHrJtY3NzI9PNDk5mJ7tIldFeuZK13ceHa5s0oe3vc3noLu4MHATB7e5P5/PNkVq9OxvPP\nk+nvb51gJMdlSU9Hm5RUYJ5pVlC+LzmRk7IUL178KUVTeDxqHfk0Plu2Bw/i8u672P7+O+dtfHnf\nNJ5Q1QsnTxtWrIinRo3cGS/3rH1PCgopS/5UWMpSWMoBuVM/yp04Ici68zZ0qDsnTtgA4OFhZt68\n63TpkiaPBciJjAxcpk3DKSSElG7d0KSlYf/dd5j8/EgcMwaLuzsakwn9qVNZY9Ls7THWr4+xcWMy\nXngBr44d8ejVC21yMihFwuTJpDdrhrl8+cePzd4ey+2TmAjxFBw+fJiQkBAsFgtNmzYlODj4jm0i\nIiJYt24dGo2GMmXKMGLEiDyINPdk1K7NtW3bsN+xg+Jz57Lk+CAWuk/ks8yhvN7xDTq94Uzdukb8\n/Ex4eFjk76sQQjwiSeKEALZssefECRs+/fQ6rVunYWNz56SBzzLduXPYHjhAWocOWQ+rBrBYsDly\nBPtt2zCsXYvu2jWSBw8m8d13ediWWfyXX+LVuTOmcuWIX7IEc9myT6AUQjw9FouFZcuWMWnSJDw9\nPZk4cSL+/v6ULFnSuk10dDQbN25k+vTpODk5kZCQkIcR56L/PTojvXlzbPfvx2nxYkb/+B7DtB8S\n8nFfJjKWs5THYLDQsWMaU6cm3DFZrBBCiPuTZqp4Zm3das/Rozb065fCRx+54OeXSYcOaWi1eR1Z\n3rE5eBDDmjUod3csHh4orRabEydw2LgRjdmMYe1absyejcPmzTh+9RW6mBiUTkd6s2ak9uqFsXHj\nh07gADJr1eLq/v1ZY+akNScKgTNnzuDj44O3tzcA9evX5+DBg9mSuF27dtGiRQucnJwAcHV1zZNY\nnxiNhowGDYhv0AD96dM4LlnC4LUhDDYt4Xi1DqzwGc2c0LocO2bDwoXXKVNGxswJIUROSRInnhmr\nVxu4ckXLiBHJnD6t56233DEaNXz+uRNms4aQkLhnI4FTCtvISLRJSWhr1LCOFbPdtw+Pvn1Br0eT\nmZk1GQlgsbcnpX9/TOXL4zJ1Kt4vvwxAepMmJAYHkx4YiLrLOLWHZSlW7LGPIUR+ER8fj6enp3XZ\n09OT06dPZ9vm5ti2yZMnY7FY6Ny5MzVq1HiqcT4tpgoVSPjoI5JGj8Zx6VKqrljB7KPfMLRGK9r/\nMZeXXqrMyy8bGTYsmZdeysjrcIUQIt+TJE48E5SCjz5yJiZGx/nzeo4ft8HJycLq1ddZscKARgPN\nmhW+WSc1N26gi43FVK4c2hs3cFi3DseVK9GfPQuAD5BZsSLmYsWwjYzEXKYMcWvXYvH0RJOaCoCy\nsQFbWwAyatbEsHo1aV26kFm9el4VS4hCwWKxEB0dzZQpU4iPj2fKlCnMmTMHR0fHbNuFh4cTHh4O\nwKxZs/Dy8nqk8+n1+kfeN9d4ecHHH2OaMgXtF19QZu5cojKrE+k/mDf+mkKPHkUICTHTpcv9nzGX\nL8qSS6Qs+ZOUJf8pLOWA3CmLJHHimXDihJ6YGB116hhZt84AwPLlcQQEZBAQUAh/9TWZcFy+HOc5\nc7JmZnRwQGMyocnMJMPfn+tvvYVTQABpGzZge+QI2pgYMurX58Ynn2D5390DdVtDEsBUtSqJ77//\ntEsjRIHj4eFBXFycdTkuLg6P2+5Ye3h4UKFCBfR6PUWLFqVYsWJER0fj6+ubbbugoCCCgoKsy486\nO1u+m9lt0CC07dvjPHcuASsX86tDKJ+VeJd+vd/m1Kk0Bg5MuefY5HxXlscgZcmfpCz5T2EpB8js\nlELc14wZzri5KYYOTeann7LGWS1adJ1du+xJS9MUyjtvAPo//8Rt+HBsjx0jvXFj0tq2xebECbC1\nJbVzZ0yVKgHg6OVFcpkyeRytEIVT+fLliY6OJiYmBg8PDyIiIhg+fHi2berUqcO+ffsIDAwkMTGR\n6Oho6xi6Z4XFy4uEDz4g5bXXcJk2jbd3jyXI7VvaTV/B+vWlmTXrBrVq5c5jCYQQojCRJE4USufO\n6fjiCydsbaFz51R277ajSpVMvL0t9OiRmtfhPTGG5ctxnTYNi5MT8f/5D+mvvAIaDWl5HZgQzxid\nTkf//v2ZMWMGFouFwMBASpUqRVhYGOXLl8ff358XXniBI0eOMHLkSLRaLb169cLZ2TmvQ88TpooV\niQ8NxWHdOqpMmcJJ+xcZfGUFHTq0Y/r0BPr0Kbx/t4UQ4lFIEicKpZAQR/R6yMyETz915uBBWwYP\nTs7rsB5eZia2Bw9it28fGbVrYwwMzFpvNmNz+DD2P/2Exc2NtPbtcVqwAKfFi0lv0oQbH3+MpZD0\nGxeioKpZsyY1a9bMtq5r167W/2s0Gvr27Uvfvn2fdmj5k0ZDWpcuZNSvj/vAgSw/2p6Xy01l4MR3\nOXVKz7Rpic/G5FNCCJEDksSJQicxUcOaNQZefTWNjAwNy5dnje1q3LjgdJ/UnTmD48qVOKxbh+76\ndev6lB49sBQpgiEsDN2VKyiNBo1SuEydisZiIaVfPxKmTQOdLg+jF0KIR2cuWZJr336L27hxDNgw\nlWIvptA65ENSUrTMmXND/rwJIQSSxIkCbNMme376yZ5z53QUL24mKMhIuXImtm+3JyVFy6BBKQBs\n2eKAwWChdu38M4GJ9vLlrCn1b32mmtmMzbFjOC1ciMPWrSi9nvSWLUkLDsYYEIDTokU4ff45AMbA\nQBInTya9cWN0//yDUzCY2wAAIABJREFUISwM83PPkfLaa4/0nDYhhMhXHBy48emnKAcHXlk5m111\nTDRdOxeLBT755EZeRyeEEHlOkjhRIO3Z8//s3Xd4lFXax/Hv9JRJr4RusAEihEgJWCCAYKOINMVV\nXMSO2EHsoFjALqJSFgtiVxRcBdZFwQIqqOjyShMIgZBepz7z/hEJICgIgUkmv8915WKemvswkDz3\nnHPuY+eaa+JJTPSTnu5jxQoH778fUXO8a1c3p5xSPRl+4MBKnM7A7ir5QWXevp3oSZOIeP99XGec\nQfHUqdh++QXnCy9g++47zJWVGFFRlI0dS8Xll9es4QZQNmEClcOHE7DbMRo3rtnvi42l9P77g9Ec\nEZGjx2ymZMoUMJvp+fLjLM300/OtJ2jd2suddwY7OBGR4FISJ/VOebmJW2+N5bjjfHzySR7h4WAY\n8OOPNnbtqp4w0b79nmpmzzxz7D+1NVVW4vj8c+wrV2JERRGIiMC+YgVhy5YBUDFsGOHvv09Kly6Y\n/H58TZpQOWwY3lNOwdWnD4HY2APe19+y5bFshohIcJnNlDz0EFgs9JjzFG8eZ2XopMfIyvJxyinB\nDk5EJHiUxEm94HbDM89EUVxsYvNmKzk5Ft59N5/w8OrjZjOcemqQy1AHAti/+oqIV18lfOFCTG43\nAZsNk7c6rt2JWvmYMfibNaP8uutwTp+Ot0MHKgcPBpstuPGLiNRFJhMlkyaBYTB47jRuSWnNyJGj\nWLLETELCXy8KLiISqpTESZ3ncsHo0fEsXRqG02lQXm7mqqvKOe20ICVtgQD2b77BsXgx1i1bKB8z\nBu/JJxN7++1EvP02RnQ0lcOGUXXOOXg6dQLDwFxSgpGcvM98NX/LlpQ88khw2iAiUp+YTJQ88ADW\nDRt48Jtr+SxwKhMmtGHGjKKDXysiEoKUxEmd5vfDqFHxLFvm4OGHi7nkkkpcLnA4ghdQ7I03EvHW\nWwSsVgJOJ+EffogvLQ3r9u2UjRtH+bXXEtjdRfg7IywsSAGLiIQIq5Wi558nsV8//l1+EU0+/JH3\n3w+jf39XsCMTETnmtOKK1GkffhjGf/8bxqRJJVxySfVir2FhQSrA6Pdj+ec/iXjrLcrGjmXHTz+x\n8+uvKb/6arBaKZgzh7JbbtkvgRMRkdphxMdT/MwzxBZvZkbSBCZMiCUnR48yItLw6Cef1Fl+P0yb\nFsWJJ3q59NLKY/qNTaWl++4LBIi57TYsr71G6W23UXbbbQSiogg4nZROnEjel1/i7t372MUoItJA\neU47Df+VVzKi4Fnaub7h2mvj8PmCHZWIyLGlJE7qnA0bLJSXm3j//XDWr7cxblwZ5mP1L9UwiLvq\nKlJbtybxggtwPvUU1p9/JurBB4l8/XX8d95J+dixxygYERE5EP+kSRjJybydMJpvV1p59NGoYIck\nInJMKYmTOuX7722ceWYybdumMmFCDCef7OXcc4/dfIeohx8mfOFCqvr3B7+f6IcfJrl3b6Kee46K\nf/wD/113HbNYRETkT8TEUHLPPSTm/MQznWfy3HNOfvlF0/xFpOHQTzypUx5+OJq4OIMhQ6r46is7\nt99eWnu9cD5f9WQ6i2W/Q6bycpzTpxP1zDNUXHJJ9QKzJhPmnTsJW7oUc1ER5WPGkBiUyXgiIvJH\nrvPPxzNjBlf8dh/3RV3C/fdH89prhcGZMy0icowpiZM644sv7Hz+uYN77inhyisravfmgQAJI0Zg\n3bCB0okTqRowAEwmTKWlRM6ahfPFFzEXF1N1/vnV6xH9/hRgpKRQOXx47cYiIiJHzmSidMIEEocM\n4eU+0+jzyUSWLHHQq5c72JGJiBx1SuKkTqiqgilTomnUyM+ll9ZyAgeEv/MOjuXL8aWlEXfddUQ9\n9BD+xo2xrVuHuaQEV69elI0bh7d9+1r/3iIicnR4unXD1bMnPb+ZSpsW13P//dH06LHrQAMuRERC\niubESdAUFZlYssTBvfdGk5mZyvff27n11lJqa0k1+xdfELZoEabCQqInTcLTvj15X35J8WOP4cnM\nBLMZ9+mns2vRIgr/9S8lcCIi9VDZ2LFYiouZ3ul5NmywsWRJsBYSFRE5dtQTJ0Hx7bc2hgxJxOUy\nYbUG6NvXxeWXV9Cli6dW7m9fsYKEiy/G5PMRsNvB66Vw9mywWqkcPlxDJEVEQoQ3MxNPZiZdvnyO\nxqnjmDnTSZ8+GlIpIqFNSZwcc4YBd90VQ2yswdNPF9G+vZeIiMBh389UWUnYggVEvPkmRmwsrr59\nibnnHnwtWlB2xx2Ev/MO3tat1dMmIhKiyseMIX70aB4e8DqXvHcp69ZZOfFELR4nIqFLSZwcc2+9\nFc6aNXaefLKIrKxD73mzbNiAbe1aLPn5+JOS8J14ImELF9YUJfGmp2P99VfCFy3CHxdH4b/+hb9F\nC1z9+h3F1oiISLC5zj4bX4sWDNz4JGFhI5k5M5JHHikJdlgiIkeNkjg5pn76ycqUKdF06OBh0KCq\nQ77OvnIlCYMHY/Lt/8mqq3dvyq++Gk+nTuD1Evaf/+Br3hx/ixa1GLmIiNRZFgvl//wnsRMncsvZ\nn/PY2925885SYmIOf5SHiEhdpiROjomNGy3ccEMc339vJzzcYNaskkNe/82cl0fcmDH4mzShcMYM\njNRULDk5WH/5BV+bNnhPOWXPyXY7rrPPPjqNEBGROqtq8GCiH3yQq/zPMcl1Bu++G85ll1UGOywR\nkaNCSZwcdXl5cMklCZSWmnjggRIGDaokNvbQPh21bNpE3I03YiopoeCVV/C1bg2AkZiI99RTj2bY\nIiJSjwSioqi68ELS3niDbifl8tprifzjH5Va/FtEQpKWGJCjJifHzH//62DQICs7d5qZO7eQUaMq\nDi2Bc7uJueUWks88E9tPP1E8bVpNAiciInIgFf/4Bya3m/tavMTatTZ+/NEW7JBERI4K9cRJrQsE\n4NFHo3jyySgArNYAM2YUkZHhPeQbxN5+OxFvvkn5P/9J+bXXYiQnH8WIRUQkFPhOPhl3586cvnYm\n4Y7xvPpqBO3aqcCJiISeQ0riVq9ezezZszEMg+zsbAYMGLDP8fz8fJ599lkqKiowDIMRI0aQkZEB\nwLvvvsvSpUsxm81cfvnltFeZ95AWCMCUKVE880wUgwdXMnx4JV26RAOuQ76H87nniHjzTcpuuomy\nm28+esGKiEjIqRw2jLhx4xjbcwXPvNedu+8uJTJSBU5EJLQcdDilYRjMnDmTCRMm8Pjjj7N8+XK2\nbdu2zzlvv/02Xbt25ZFHHuHGG29k5syZAGzbto0VK1Ywbdo07rzzTmbOnIlhGEenJRJ0gQA8+GB1\nAnfJJRU8/ngxXbp4SEw89HuEv/ceUQ89RNUFF1B2001HL1gREQlJrt69CVitjIp9i/JyM+++Gx7s\nkEREat1Bk7j169eTmppKSkoKVquVrKwsVq5cuc85JpOJysrqClCVlZXExcUBsHLlSrKysrDZbCQn\nJ5Oamsr69euPQjMk2AIBmDQpmueei+LSSyt46KFDqz5pW72alA4diB07loh584gdOxZP584UTZuG\nZqOLiMjfFYiLw52VRcvvP6RNaw9z5kQSUEeciISYgz5mFxYWkpCQULOdkJBAYWHhPudcdNFFfP75\n51x11VU89NBDjBo16oDXxsfH73ethIZHH43i+eedXH55OQ8++OcJnG3NGpI7dSJi7lzMhYXEjR4N\nfj9hH35I7C234DvxRApnz4ZwfXIqIiKHx9WvH9ZNm7ip77f88ouNVatU4EREQkutFDZZvnw5Z511\nFueffz7/93//x9NPP83UqVMP+frFixezePFiAKZMmULi3xl/9wdWq/WIrq9L6ktbnn/ezJNPWhk1\nys9zz9kxmfaNuaYdgQDW++/HtH07sePHE3jqKSgsxPfZZwTS0jC/9RaBIUNIqMNFTOrLe3Io1Ja6\nSW0ROXKus88mMGEC/f3vcmNUZ+bOjeS004qDHZaISK05aBIXHx9PQUFBzXZBQQHx8fH7nLN06VIm\nTJgAwAknnIDX66WsrGy/awsLC/e7FqBXr1706tWrZjs/P//vt+R3iYmJR3R9XVLX2/LTT1beeCOC\nWbMi6d3bxT33FLLX211jdzvCFiwg/quvKH74YSzbt+N8+mlKHnmEymbNqk8cNqz6zzrc5rr+nvwd\nakvd1NDakpaWdoyikYbESEnBk5lJ9OKFDBlyNy+/HMk995SSmKh5+SISGg46nDI9PZ3c3Fzy8vLw\n+XysWLGCzMzMfc5JTEzkp59+AqqLmXi9XqKjo8nMzGTFihV4vV7y8vLIzc2lVatWR6clckzNnRvB\n2WcnM3duJAMHVjF9ehHWv/hIwFReTvSDD+I9+WQqhw+n7Lbb2PF//0fl8OHHLmgREWkwXH37Ylu7\nltF91uHxmHj99YhghyQiUmsO2hNnsVgYNWoUkydPxjAMevToQdOmTZk/fz7p6elkZmZy6aWXMmPG\nDD766CMArrnmGkwmE02bNqVr167cdNNNmM1mrrjiCsyHUu1C6rRAAGbMcNKhg4e5cwuIj99rxrhh\nYN61q3pdN5MJ8/btmGfNInnaNCxFReTPmwcWS/V9NO9NRESOElevXsQ88AAnbviEbt3a8vLLEVx9\ndfnuX0EiIvXaIc2Jy8jIqFn3bbehQ4fWvG7SpAkPPPDAAa8dNGgQgwYNOoIQpa758ks7mzdbGTeu\nqCaBs2zZQvgHHxAxbx7WzZsxoqPxJyVh27ABAFfPnhSOG4f3D/+OREREjgZ/ejq+5s0JW7KEf/xj\nDFdeGc/SpQ5693YHOzQRkSNWK4VNpGGZNy+C6GiDc891YVuzhthx47CtWweAu0sXKkaOxLppE5bt\n26kaNoywiy6iMCkpyFGLiEiDYjLhys4m4rXXOPvZIlJTY5g7N1JJnIiEBCVx8rcUF5v46KNwhg2r\nJDw8QPT992MuLKTk3ntx9e6Nv0WL/a4JS0ys08VKREQkNLmzs3HOmkXkyi8ZMSKZxx93sn27mbQ0\nFTgRkfpNE9Tkb3nllUjcbhMjRlRg++47HF99Rfk111AxevQBEzgREZFgcXfpghEeTtiSJVxwQRWB\ngIklS8KCHZaIyBFTEieH7NtvbTz2WBS9e7to29aHc/p0jJgYKkeMCHZoIiIi+wsLw9O9O44lS2iV\n7qVZM5+SOBEJCUri5JAUFJgZMyaeRo38PPFEEda1awlbtIiKkSMJOJ3BDk9EROSAXD17Yt26FduG\n9WRnu/j8cztVVcGOSkTkyCiJk0Oy4op3+Tz3RFY7szj+ol4k9+lDIDycilGjgh2aiIjIn3JnZwPg\nWLKE7Gw3LpeZL790BDkqEZEjoyRODip/h8E5Kx8kyuknIs6OERdH6Z13smvxYoyUlGCHJyIi8qf8\njRvjPflkwpYsoWtXN+HhhoZUiki9p+qUclDf3/9f/sE21t4+C9+os4MdjoiIyN/i6tkT54wZhHtK\nOf30OJYscTBpEphMwY5MROTwqCdO/pLHA8ctnMlORxPiLs0OdjgiIiJ/mzs7G5PPh+Pzz8nOdrN1\nq5VfftHn2CJSfymJk7/0xYtbON37H3L7XwZW/cITEZH6x9OxI0ZMDGFLltC3rwuzOcCCBeHBDktE\n5LApiZMDMwzCPvqIbk+Mwo2d5AlDgh2RiIjI4bFacZ95Jo6lS0mM99G9u5sPPggnEAh2YCIih0dJ\nnBxQ7LhxxF95JZ5KP/86dy4kJQQ7JBERkcPmys7GsmsXtp9+on//KjZvtvLDD7ZghyUicliUxMl+\nwj76iIi33uKbM6+nNT/T4oYewQ5JRETkiLh79CBgMuH4fUilzRbg/fc1pFJE6iclcbIPc0EBMePH\n4znlFO7wT6ZZiwBt2viCHZaIiMgRMRIS8LZvT9iSJcTGBjjzTDcLFoRhGMGOTETk71MSJ3sEAsTc\ncQfm0lJ+u+8pln3p5LzzqlSCWUREQoIrOxvb6tWYCwoYMKCK7dutfPqp1owTkfpHSZzUiHjtNcIX\nLqTsttt4f0N7/H4T553nCnZYIiIitcKdnY0pEMCxdCnnnVdFerqXyZOj8HqDHZmIyN+jJE4AMK/7\nlei778Z9+ul82e16HnwwiuOP99K2rX6ziYhIaPC2bYs/OZmwpUux2WDixFI2bLDx6qsRwQ5NRORv\nURInAGwfPplCVyRX2OYwZFgSEREB5swp1FBKEREJHWYz7h49cPz3v+Dz0bu3m65d3UydGkVZmX7h\niUj9oSROqNyUT7udS3kr4Z/8+4fmNGrk5513CmjRwh/s0ERERGqVq2dPzCUl2L//HpMJbrmljMJC\nC//5jyPYoYmIHDJrsAOQ4Mt54mNa4eeEu89h9YU71fsmIlILVq9ezezZszEMg+zsbAYMGLDP8c8+\n+4yXX36Z+Ph4APr27Ut2dnYwQm1Q3N26ETCZsH/+OZ7TTiMz00NUlMHy5Q4uuEDzwEWkflASJyQu\nfpdfrG05YVArJXAiIrXAMAxmzpzJxIkTSUhIYPz48WRmZtKkSZN9zsvKyuKKK64IUpQNUyAuDm+7\ndjg+/5zym27CaoUuXTx88YV64kSk/tBwygau8pcc2hR/xf/aD8Ksfw0iIrVi/fr1pKamkpKSgtVq\nJSsri5UrVwY7LPmd+/TTsX/3HabycgC6dXOzebOVnBxLkCMTETk0emxv4HY++REA0VeeF+RIRERC\nR2FhIQkJCTXbCQkJFBYW7nfe119/zS233MLUqVPJz88/liE2aO7TT8fk82H/6isAund3A/DFF/Zg\nhiUicsg0nLKB+uorO6/MsfPYR6+wyt6Fk85pHOyQREQalI4dO9KtWzdsNhuffvopzz77LPfcc89+\n5y1evJjFixcDMGXKFBITEw/r+1mt1sO+tq454rb07UsgLIzYlSvxDxtGt26QlBRg1aporr02svYC\nPQR6X+omtaXuCZV2QO20RUlcA3X33TGcsWEuxxkbKLrjHs2FExGpRfHx8RQUFNRsFxQU1BQw2S0q\nKqrmdXZ2Nq+88soB79WrVy969epVs324PXaJiYkh09tXG22J79QJy6ef1tyna9c4li61s2tX/jH9\nnaj3pW5SW+qeUGkHHFpb0tLS/vK4hlM2QIEAbN1ocJ9tEp527Wh0Zc9ghyQiElLS09PJzc0lLy8P\nn8/HihUryMzM3OecoqKimterVq3ar+iJHF2e00/Htm4d5txcoHpe3I4dFtat0+fbIlL3KYlrgHbu\nNDO46hWSyjZTdtNNqBtORKR2WSwWRo0axeTJkxk3bhxdu3aladOmzJ8/n1WrVgGwaNEibrrpJm69\n9VYWLVrENddcE+SoGxZX794AhC9YAECfPi4cjgAvvXRsh1OKiBwOfdzUAG3caOUmplHY8lTcew3R\nERGR2pORkUFGRsY++4YOHVrzesSIEYwYMeJYhyW/8x1/PJ527Qh/+20qrryS5GSDiy+uYO7cSMaO\nLadpU3+wQxQR+VPqiWuASr/8lTb8TOmFQ9ULJyIiDVbVhRdi/+knrOvWAXDNNeWYzfD0084gRyYi\n8teUxDVAKf/9AD9mbMP6BTsUERGRoKkaMICAxUL4O+8A0KiRwYgRlcyfH8Hy5VpuQETqLiVxDU0g\nwCn/e5eV4WdgapQc7GhERESCxkhMxH3mmdVJnGEAcP31ZaSl+RkyJJHrr4+lslIjVkSk7lES18BY\n//c/mlWsY2XLgcEORUREJOgqBw/Gun079q+/BiA11WDp0l3ccEMZ77wTwZw5KnQiInWPkrgGxvHB\nAvyY2Xra+cEORUREJOjcvXoRCAsjbNGimn3h4QFuv72M005zM29eBIFAEAMUETkAJXENid+P4613\n+Q89SGoTF+xoREREgi4QGYnrzDMJX7iwZkjlbiNGVLJxo5WvvtL8OBGpW5TENSCO//yHsO1beIEr\nOe44X7DDERERqRNc/fphyc3FtmbNPvvPP99FVJTBa69FBCkyEZEDUxLXgET+61+URaXyLgNp2VJJ\nnIiICFQv/B2wWglbuHCf/eHhAQYOrGLhwnCKi1XgRETqDiVxDYRl0ybCli5laatR2CMspKQYB79I\nRESkAQjExuLu1q16SOUfJsBdckkFbjcMG5bA+vWWIEUoIrIvJXENROTcuQSsVl6J+CctW/q1xreI\niMheXOecg3XzZmw//bTP/jZtfMycWcTWrVb69k1ixQrNjxOR4FMS10CEffwxruxslm9sxvHHe4Md\njoiISJ1Sde65GOHhRM6cud+xs892sWRJHo0aGdx4YyxlZfokVESCS0lcQ+B2Y9m2jaJmbcnNtZCR\noSRORERkb4G4OCqHDSP8vfcw79ix3/HUVIMnnigiN9fCffdFByFCEZE9lMQ1ANYtWzAZBv8zjgcg\nI8MT5IhERETqnop//hN8PiJnzz7g8Y4dvVx9dTnz5kWybJnjGEcnIrKHkrgGwLpxIwDfFJ2EwxGg\nTRv1xImIiPyRv0ULXP36EfnKK5gqKg54zs03l9G0qY9HHonSIuAiEjTWQzlp9erVzJ49G8MwyM7O\nZsCAAfscnzNnDmvXrgXA4/FQUlLCnDlzABg6dCjNmjUDIDExkdtvv70Ww5dDYdm0CYBPN59E27Ze\n7JqTLSIickDlV11F+MKFRD/4ICWTJ+933OGA664r5/bbY1m2zMGZZ7qDEKWINHQHTeIMw2DmzJlM\nnDiRhIQExo8fT2ZmJk2aNKk557LLLqt5vWjRIjb9njQA2O12Hn300dqNWv4W68aN+OPjWfFzMiNH\nHviTRREREQFvx46UX3klzhdewN2tG65zztnvnIsuquTJJ51MmxbFGWe4VfFZRI65gw6nXL9+Pamp\nqaSkpGC1WsnKymLlypV/ev7y5cvp3r17rQYpR8a6cSOlqem4XCbNhxMRETmI0vHj8Zx6KrG33II5\nJ2e/4w4HXHttOatW2Vm+XMNbROTYO2hPXGFhIQkJCTXbCQkJ/Prrrwc8d9euXeTl5dG2bduafV6v\nlzvuuAOLxUL//v3p1KnTftctXryYxYsXAzBlyhQSExP/dkN2s1qtR3R9XVJbbbFt2cKGxj0ByM52\nkpjoPOJ7/h16T+omtaVuUltE6gC7naLnniO5Z0+ip06leNq0/U4ZNqySadOimDMnku7d9QGpiBxb\nhzQn7lAtX76cLl26YDbv6eB77rnniI+PZ+fOndx///00a9aM1NTUfa7r1asXvXr1qtnOz88/7BgS\nExOP6Pq6pDbaYqqspFFODqtj00lO9hMRkc+x/uvRe1I3qS11U0NrS1pa2jGKRuTv8bdoQcXIkUTO\nmkXZNdfgb9Vqn+NhYTBkSBUvvhjJzp1mUlKMIEUqIg3RQYdTxsfHU1BQULNdUFBAfHz8Ac9dsWIF\n3bp12+96gJSUFFq3bs3mzZuPIFz5u3YXNVm+60QyMjwaty8iInKIyq+7joDDQdQBeuIAhg+vwOcz\n8cYbEcc4MhFp6A6axKWnp5Obm0teXh4+n48VK1aQmZm533k5OTlUVFRwwgkn1OwrLy/H660uZ19a\nWsq6dev2KYgiR9/u5QW+yDtZi3yLiIj8DUZSEhVXXEHE++9jW7Nmv+Pp6X66dnUzb14EhjriROQY\nOuhwSovFwqhRo5g8eTKGYdCjRw+aNm3K/PnzSU9Pr0noli9fTlZWFqa9unpycnJ44YUXMJvNGIbB\ngAEDlMQdY9bfe+LW04qMjKogRyMiIlK/lF99NRFvvUXcVVexa+FCAnFx+xy/+OJKrrsujqVLHfTq\npeUGROTYOKQ5cRkZGWRkZOyzb+jQoftsDxkyZL/rTjzxRKZOnXoE4cmRsm7cSHFkI1xVEZx6ammw\nwxEREalXArGxFL7wAomDBxN37bUUvvwyWCw1x/v1q6Jp0yjGjInj8ceLueACVxCjFZGG4qDDKaV+\ns27axCbr8Zx8so+IiECwwxEREal3vB07UjJpEmH//S8xt9/O3mMnw8Lggw/yOeUUL1dfHc/8+eFB\njFREGgolcSHO8ttv/FjZSuvDiYiIHIHKiy+mbOxYIufN2y+RS042eOONAjp29DB1ahReTUEXkaNM\nSVwoq6rCsmsX67zHKYkTERE5QmW33lqdyL32GlEPP7zPMbsdbrihjJwcK++/r944ETm6lMSFMEtO\nDgC/0VxJnIiIyJEymSi79VYqRowg6plncHzyyT6Hs7PdnHSSl+eec6papYgcVUriQph12zYA8iOb\ncdxx/iBHIyIiEgJMJkoeeABP27bE3Xgjlr3WvzWZ4Oqry1m3zsZLL0VSWanFWUXk6FASF8Isvydx\nUW0bYdY7LSIiUjvCwih64QUwmUi45BLM+fk1h/r3r6JNGy/33RdDu3YpvPaaFgIXkdqnR/sQZtm6\nFS9WEtslBzsUERGRkOJv3pzCOXOw5OYSP3IkpvJyAGw2WLRoF2++mU+7dl7uuSea3Fw9bolI7dJP\nlRDm3ZDDVprStIWWFhAREaltntNOo/D557GtXUv8FVeAu3qxb4sFsrI8PP54MX6/iQcfjA5ypCIS\napTEhbDApm1spgXNm2s+nIiIyNHg7t2b4scew/HFF8SNHQv+Pb9zmzf3c+WV5bzzTgSrVtmCGKWI\nhBolcSHMkbuN32hOs2a+YIciIiISsqqGDKFk4kTCFywg5u67IbBnBMz115eTmurnrrtiVLFSRGqN\nkrhQ5fEQUbKD32hOkybqiRMRETmaKq6+mvKrriJyzhycTzxRsz8yMsCdd5byww925s9XkRMRqR1K\n4kKUZft2zAQoiW2KwxHsaEREREJf6Z13Ujl4MNGPPUb03Xdj3rkTgIEDq8jM9DBlShSlpVp2QESO\nnJK4EGXZuhUAd6OmQY5ERESkgTCbKX7sMSpGjCBy9mxSunTBOW0apoDBAw+UUFBgZtCgRCZOjOan\nn6zBjlZE6jElcSHKkpMDgLllkyBHIiIi0oDYbJQ8+ih5n39OVb9+RE+dStyYMZyaXsyUKSU4nQav\nvx7ByJEJlJXja1sXAAAgAElEQVSpV05EDo+SuFC1aSt+zESemBLsSERERBocf4sWFD/7LCX33EPY\nxx+TMGIEIwfl8957Bbz5ZgG7dpl54omoYIcpIvWUkrgQ5fl1Ozk0pulxeotFRESCwmSi4sorKZo+\nHdt33xE3ejR4PHTo4GX48EpeeimSX34JdpAiUh/pCT9Ubd7KZlpoeQEREZEgc513HsWPPkrYZ5+R\ndO65RLz8MhOu205kZIDLLrNSWFg9rDIvz8zOnXo0E5GD00+KEBW2Yyu/0VwLfYuIiNQBVcOGUfTs\nsxAIEHvHHZzcN5MvTr+Jgp/zGDw4kQcfjKJr1xQuuCARjyfY0YpIXackLgSZqqqIKslhs7UViYla\nWVRERKQuqBowgF2ffsquBQtwn3EGbRY+zUbL8Qzd+CgvPWujY0cP27ZZef11rScnIn9NSVwIsmzY\ngJkABcknYFLhKxERkbrDZMKbkUHRjBnkffYZ5t49uc97J0UpJ/DBJbPpmOHmqaeicLuDHaiI1GVK\n4kKQdcMGAKqatwpyJCIiIvJn/Onp+N58k/w33sCaEEX81VezqOJMGud+x6xZkRpWKSJ/SklcCDKv\nW48fM/Y2LYMdioiIiByEp1s3dn38McWPPUZ84UZW0ok2k8ZwcastXH11HIZmRojIHyiJC0FV321g\nEy1pm6m3V0REpF6wWKgcPpy8L74g/8rrGRT2Ecv9XZn8wWnsGDgRx5IlwY5QROoQPeWHIPOv6/kf\nJ5GR4Q12KCIiIvI3BJxOPPfcQcGabymeNJmq+DSOX/U2CZdeSvRdd6ExliICSuJCj99PbN4Gtkac\nQFqalhcQERGpjwJOJ5WXX4bl07mkR+cxL/UGnLNmkXj++YTPn4+poiLYIYpIECmJCzGWbdtwGC5c\nxx2vypQiIiL1XGqqwc13VDFix5N8ccNLmCsribvpJlIyM3E+8QSm8vJghygiQaAkLsRUfV9dmTK8\nQ3qQIxEREZHaMHx4JWlpPsatGMnO/y4j/733cHftSvSjj5LSuTPRkyZh2bo12GGKyDGkJC7EFCzf\nBECjs1oENxARERGpFXY7XHddOatW2Vn2eRifebsxf9ir7PrwQ9zduhH5wgskZ2URN2oUjmXLwK/p\nFCKhTklciPH+sIE8kji5e1SwQxERadBWr17N2LFjuf7663nvvff+9LyvvvqKIUOGsOH3NT5FDmTY\nsEoaNfJzySXxXHRRIpdfnsDS8s4UvfACO7/8kvJrrsG+ciUJw4eTkpFBzO23Vyd0XhU5EwlFSuJC\nTMTWX9kSfiJOZyDYoYiINFiGYTBz5kwmTJjA448/zvLly9m2bdt+51VVVbFo0SKOP/74IEQp9YnD\nAZMmldC3r4unniqiRQsft90WS2WlCaNxY8rGj2fnypUUPv88nqwswt95pzqh69CB6Lvvxrp2LQT0\nbCASKpTEhRCfN0CT0v9RmtYq2KGIiDRo69evJzU1lZSUFKxWK1lZWaxcuXK/8+bPn0///v2x2WxB\niFLqm759Xbz4YhEXXljF1KnFbNli5eGH9xp5ExaG6/zzKZo+nR0//EDhzJl4unUjcu5ckvv0IeXU\nU4kbPRr7V18FrxEiUiuUxIWQte9uJT5QiK1b+2CHIiLSoBUWFpKQkFCznZCQQGFh4T7nbNy4kfz8\nfDIyMo51eBICunTx8I9/VPDSS06eeMK5fydbeDiuvn0pmjGDnd99R/Gjj+Lu2RP7N9+QeOGFJAwe\nTOSMGdi+/149dCL1kDXYAUjt2fH2twA0G9EhyJGIiMhfMQyDuXPncs011xz03MWLF7N48WIApkyZ\nQmJi4mF9T6vVetjX1jVqS7VnngGv18+jj0ZTXBzJk0/6sVgOcGJiIpxwAgD+qioCL76Iffp0HPff\nD4DRoQP+8eMJ9O1bPW7zMOl9qZtCpS2h0g6onbYoiQsRgQBEfPsNJbYEbG21vICISDDFx8dTUFBQ\ns11QUEB8fHzNtsvlYuvWrdx3330AFBcX88gjj3DbbbeRnr7vz/BevXrRq1evmu38/PzDiikxMfGw\nr61r1JY9HnkE4uOjeOaZKLZs8fDss8Vs3Wph61YLPXq4MR9ozNWIETBiBOa8PMIWL8b57LPYhgwh\n4HDgOfVUPKedhiczE09WFgGn85i1pS5RW+qeUGkHHFpb0tLS/vK4krgQ8cMPNjKqVrCrbWcitMq3\niEhQpaenk5ubS15eHvHx8axYsYIbbrih5nhERAQzZ86s2b733nsZOXLkfgmcyMGYTDB+fBmpqX7u\nuiuGjh1TKCmpztwGDKjk8ceLsdsPfK2RnEzliBFUDhlC2JIl2L/6CvvKlThfeAHTs88SsNlwZ2VR\nOXQornPOAc3dFKkzlMSFiOVvl9CP9eT2HYFGtouIBJfFYmHUqFFMnjwZwzDo0aMHTZs2Zf78+aSn\np5OZmRnsECXEXH55JY0aGbzxRjjdu3soKTHx2GPR5OdbmDWrkMjIv3g6sFpxnX02rrPPrt6uqsL+\n/feELVlC2MKFxF9zDf5Gjajq1w9Ply54OnfGCJFhbSL1lZK4EFGysHo+nPXM09CKMCIiwZeRkbFf\n0ZKhQ4ce8Nx77733GEQkoa5vXxd9+7pqtps08XPzzbFcfHE8L79cSFTUIX7MGx6OJysLT1YWpXfe\niWPJEiLnziVi3jycs2YB4D3+eHwnnIC/SRPcXbrgPuuso9AiEfkzSuJCwI8/2kjP/RKvLRxv27bB\nDkdERETqgIsuqiI8PMC118Zx4YWJ9OjhonlzP4MHV/7pEMv9mM24e/fG3bs3eDzYfvwRx9dfY//6\na6z/+1/1nLoZM/DHxcHQodjOPRdvhw7V4zxF5KhREhcCXn89gutMX+Bp34FD/6ksIiIioe6881zY\n7YVMnBjD9OlO/H4TX31l58kniwHYuNFCerr/0G5mt+Pt2BFvx46wu7Kq14vjs8+IePttwubMIen5\n5zFiY/G2bYuvWbOa166zz4awsKPUSpGGR0lcPedywY43V9Eh8B3lZ90S7HBERESkjunTx02fPnkY\nBjz5pJPHHovGYoF166ysWWNnxoxCzjvPdfAbHYjNVtNTl2izUfHKK9i//Rbbjz8Stngx5uJiTB4P\nRmwsrrPOwnfCCXi6dsVz2mnqrRM5Akri6rnF7xs8XXEFlcnNqBg9OtjhiIiISB1lNsONN5azY4eF\nV16JpHFjHykpfmbOjDz8JG5vMTFUDR1K1d5zPw0D+xdfEDF/PvZVq4h47z0AvK1b4+rbFyMyEn/z\n5rh69FBPncjfcEhJ3OrVq5k9ezaGYZCdnc2AAQP2OT5nzhzWrl0LgMfjoaSkhDlz5gDw2Wef8c47\n7wAwaNAgztLE11pRUhQg94ttNJr8LK3YQN4zbxGIjAx2WCIiIlKHmUzw4IMlnH9+FZmZHubMieSB\nB2L4+WcrrVv7av8bms14zjgDzxlnVH//8nLCP/iAyFmziJo2reY0IzoaV3Y23owMvG3a4GveHCM5\nmQMvdCciB03iDMNg5syZTJw4kYSEBMaPH09mZiZNmjSpOeeyyy6reb1o0SI2bdoEQHl5OW+99RZT\npkwB4I477iAzMxPn31g48m8zDKLvvhuoXgC7rNREQYEZj8dEIFC9zzAAE5hN1T/MTCYwmQOYAJO5\ner9hgNdb3c1vtQUIBKCy0ozXu9d1v59rNgcwmap/zuy9H9NeVaACpurS/7/v2uf1H4pF7T6202bD\n4/XC73G7XCZcZX5ic/+Pk13fczKlAHzZ9Vqad+t6NP42RUREJMRYLNC9uweAoUMrefTRaP71r0im\nTClhxw4zKSnGUcudAk5n9dp0I0aAz4fp9+UMwt9+G8eyZUS8++6ec8PC8DVrhr9JE/zJyfhbtKBq\nwAD8TZseneBE6pGDJnHr168nNTWVlJQUALKysli5cuU+Sdzeli9fzpAhQ4DqHrx27drVJG3t2rVj\n9erVdO/evbbiP6CIt94CoKzMRLgBB460njKZ2BHVip9Pvgij/SmEd29L87NPCnZUIiIiUg/FxQXo\n37+Kt98O57vv7Pz8s40uXdw89lgxLVseYsGTw2W1EoiKwn3GGbjPOAMCAcy5udj+9z8sW7Zg3bIF\ny2+/Ydm2DdvPP2PeuZPoKVNwd+5cPa+uY0e8J52EkZpaPfeuqgp/48ZHN2aROuKgSVxhYSEJCQk1\n2wkJCfz6668HPHfXrl3k5eXR9vcy93+8Nj4+nsLCwiON+a+Zzez4+WcCAWjSJI1zz63ihhvKSEgw\nsFiqP30ymwO/98iZ8Pv5/WvPa8MwYbEEcDqre9gqKkyYzZCU5Cc8vPrb7O7R8/mqr/X59n1tGAea\nrBuo6fmDvXoB/2RfQkI8BQUFNb18UVEBzGaIofpLRERE5EiNGlXO22+HYzLBddeVMXduJL16JXHW\nWW46dfIwYkTloa8xdyRMJoy0NNxpaQc8bMnJIfyNNwj7+GOcTz2FyTAACJjNNa+9bdpQOXAg/mbN\nMOLjMRIS8CclEYiLO/rxixxDtVrYZPny5XTp0gXz3+yDX7x4MYsXLwZgypQpJCYmHnYMVquVxMRE\nqqqqt7t2tXHWWbGHfb9gslqt+yTB9dXu9yQUqC11k9pSN4VSW0RCWdu2Pn78cQdRUdUfNl92WQWP\nPx7F8uUOPv44nGXLHLz8cmHQp6f5GzemfNw4yseNw1Reju2HH7D+3/9h2bEDIykJfD7C33uPmEmT\n9rvWdeaZVFx2Gd6OHTHi41UZU+q9gyZx8fHVvUG7FRQUEB8ff8BzV6xYwRVXXLHPtT///HPNdmFh\nIa1bt97vul69etGrV6+a7fz8/EOL/gASExPJz8+nuNgENMLnqyA/v+Kw7xdMu9tS34VKO0BtqavU\nlrrpUNqS9iefuIvIsRUdvaenrVEjg0ceKQFg7twIxo+P5cknnYwbVx6s8PYTcDrxZGXhycraZ3/F\nmDGY8/Iw79qFubAQc2Ehtl9/JWLePBIuvxyoLqLia9kSX8uW+Js3r553t/srNRWsKt4udd9B/5Wm\np6eTm5tLXl4e8fHxrFixghtuuGG/83JycqioqOCEE06o2de+fXvmzZtHeXn1f/o1a9YwYsSIWgz/\nz7lc1Z+whIUdg+5/ERERkRA0cmQlK1famTo1irw8C1lZbrp3dxMXV3efr4zk5OrKlr9zAWVjx+JY\nvhzrr79i3bQJy6ZN2L/9FsuCBZj8e+b+BcxmjORkTE2bEpeUhD8tDX/jxngyM/G2b69qmVJnHDSJ\ns1gsjBo1ismTJ2MYBj169KBp06bMnz+f9PR0MjMzgeqhlFlZWZj26p52Op1ceOGFjB8/HoDBgwcf\n3cqUe9mdxDkcdfeHjIiIiEhdZjLBww+X4HKZePPNcObOjcRiCdC5s4errionO9sd7BAPjc2G+6yz\ncP9xqSuvF8v27XsKqeTmYs7NJTw/H+uvv+L47DPMlZUA+JOS8LVqhREVhZGcjL95c7ytW+Pp2JGA\n04mpuBjsdi35JMfEIfUXZ2RkkJGRsc++oXsv5Ag1FSn/qGfPnvTs2fMwwzt8brd64kRERESOVERE\ngBdfLMLrhTVrbCxeHMaCBeFcemkCF11UyX33lRATU0+ft2w2/M2b42/eHM/eu3cPBw8EMOfn4/j8\ncxxLl2LZvh3r1q2YV67EUlQEVPfeYbFg8noBMGJiqnvw9vrypafj6dABQ0PIpZaE7KBfDacUERER\nqT02G2RmesnM9DJuXBlPPhnFM884+eEHG6+9VkBI1jEymTCSkqgaNIiqQYP2PVRSgm3NGuyrVmFy\nuzESEzF5vVhycjBv3441Jwf7t99iLi6uucaIjsaIi/vTL39aGp7TTlM1TTmokE3i9vTEBTkQERER\nkRDjcMBtt5WRleVm1Kh4Bg5M5KOPDP6k9l1ICsTE4DnjDDxnnPGX55kqK7GuW4f9u++wbN6Muaio\n+quwEOuGDdWvy8r23NdkwnfiifhOOAEjLg7rr79iLijAe/LJeE89FW/79nhPOglsNgImkx52G6iQ\nTeLUEyciIiJydHXv7uGNNwq45JJ4OnWycfPNkYweXaECj3sJRETg7dABb4cOf36S14u5uBjrpk3Y\nly/H/t132FavxlxYiO/44/E3bYrj66+JeO+9/S41YmPxNWmCt0MHPB06YHK5MJeVYcTG4k9OxtSq\nFVa3m7B//5uwJUvwdOxIxciRmCsqsP3wA77jj8eTmVm9mPJ+wQe0HEMdFbL/xVyu6j9V2ERERETk\n6Gnf3ssnn+zigQeSmTQphmXLHMyeXUhYGPzwg42oKIOWLf0Hv1FDZrNhJCXhSUrC06nTn55mzsvD\ntmYN1vXrMQUC1YVZdu7Esnkz4e++S+TLLx/wut21Oj1t2xI5ezbOF1/c57g/ORlfejpGbCxGTAyB\nyEis69djX7kSIzkZV79++NPSMJWU4G/RAlefPvsWcHG5MFdWEggLIxAWpiqex0AIJ3HqiRMRERE5\nFtLSDN5808fTT1dy660xjB4dz3HH+XjpJSfNmvn4z3/yNOqvFhjJybh798bdu/f+B30+LL/9RsDp\nJBAdjam4GMuuXcRWVVG+ZQueLl3wN22KeccOwhcswJ+cjLddO2w//EDYv/+NJTcX68aNmEtKMJWW\n4m/WjKoLL8SybRuRL71UU7gFwAgPx3fiiQQcDsz5+Vg3bcJkGDXHAw4H/saNcfXoge+EEzAXFgLg\nbduWQFgYjs8/x7JlC0ajRviaNMHfuHH1V5MmBKKiMJWVYc7Px1JQAG53dU/hX/7FGJjz8jBSUhpM\nz6GSOBERERGpFcOHV2IYcNttsSxdCn37VvHxx+G8+KKT66+vO4uFhySrFX96es1mIDwco1EjAomJ\nVOXn1+w3UlOpGD26ZtvfsiWu/v3/8tam8nJMHg+G04n9++8J/+ADLL/9hsnlwnf88bjOPx8jIQGT\ny1XTK2ddt47IV1+t3vcHAYsFf6NGWPLyMHk8+x3be+0+AMPphOxs4qqqMFdW4mvaFH9aGpb8fCy/\n/YZ99WrMxcX4mjXDdc45mFwuLFu24D3pJDzdumHOz8f2yy/4U1PxdOiAv3FjAtHRBCIi9k/6vN7q\noaV/szfRVFGBqbISIynpb113uEI2idtd2ETDKUVERESOnYsvriQpyY/TGSAry8MVV8BTTzm56KJK\nnM4AZnP1sgWGAT/9ZKOkxMTJJ/tITDQOfnMJioDTye4nak/nzng6dz6k60xVVZgLC/EnJGDyerGt\nXYupvBxPp04EoqOre9B27cKSk4Nl2zYs27djLirCiI/HSEzESEwEr5ewRYuI+OYbrGFhBBwOwj79\nFEt+fvVyDk2aUHXOOfjS03EsW0bkCy8QiIzE37gxjmXLMD33XHUbbLZ9ehOhOmE0oqMJxMRgREVh\nLirCkpODkZSEu0cPcLtxfPEFmM14TjsNIzGxulfRMDBiY8Fmqy5cs2EDtjVrMPl8NYli0YsvHtVe\nwZBN4tQTJyIiIhIcffrsWQT8rrtK6dEjma5dU/B4qp/PmjTxUVlporBwTzGN7t3dvPZawQHra0j9\nFAgPx9+4cfXrsDA8Xbrse4LZjJGSgpGSgvcPa1Lvzd2r1561+2p2uqvLpO6l4qqrqgtjOBxgMmEq\nL8f+3Xf4U1LwtWpV3SO3Zg2WXbswlZZiLinBXFZW83p3ERnrpk2EffwxAbsd9+/VR+3ffIO5vBwj\nPp6A2Yy5uBiTz1fdxrQ0yq+6CiMhAduPP2KqqjrqwzpDNolTT5yIiIhI8LVo4WfatGK+/dZGWpqB\nxwO//mrFaoUzznCTlORn2bIwpk938vrrEVx8cWWwQ5b64A8JXI29Jl8GnM6aJAzASEnB3afPod0/\n8HsOUUfn2IVsErenJy7IgYiIiIg0cAMHVjFwYNWfHj/9dA/ffmvj4YejOP/8KqKj9SG8BFkdTd52\nC9n6ny6XCYcjUNf//kVEREQaPJMJ7r+/lMJCM5MnR/OHWhci8gch2xPndms+nIiIiEh9ccopXkaO\nrGTu3Eg++SSMvn1d2O0B4uIMzjzTzamnerX8mMjvQjaJc7lMSuJERERE6pHJk0vo3dvF7NmRvPde\nOIYBFRUmHn00mrQ0H7fcUsbgwVUqfiINXkgncSpqIiIiIlJ/mM3Qs6ebnj33VLcsLDTx2WdhzJoV\nyU03xfHUU1F06uShc2c3Q4ZUqXdOGqSQ/WevnjgRERGR+i8+PsCgQVUsWJDP9OmFNG/uY8kSBzff\nHMeYMXFUVakAgjQ8IZvEud3qiRMREREJFSYTXHCBi9deK2TNmp3ce28JixaFMXBgAj/8YKs5L6DH\nP2kAQno4pXriREREREKPyQSjR1fQvLmPm2+OpV+/JE4/3U1OjoWtWy107uyhX78qBgyoIjZWz4MS\nekK2J646iQt2FCIiIiJytPTp42b58jyuu66MrVsttGrlZeTICnbsMHPnnbFkZqZw220xLF9ux+sN\ndrQitSdke+LcboiL0ycvIiIiIqEsOjrA+PFljB9fts/+n36yMnt2JG+/HcGrr0YSG2swdGgll19e\nQdOm/iBFK1I7QrwnTkmciIiISEPUtq2PqVNL+OGHHbz0UiGnn+7mpZciycpKZvToOL75xq75c1Jv\nhXBPnAqbiIiIiDR0kZEB+vVz0a+fi5wcM3PmRPLqq5EsXBhOaqqfM890k5np4dRTPbRu7cOkYpdS\nD6gnTkREREQahMaNDe68s4xVq3YybVoRmZke/v3vMG69NZY+fZK5//5ooLrC5bJldpYvt1NerqxO\n6p6Q7YlTEiciIiIiBxIREWDo0CqGDq0iEIDNmy1Mn+7khRecNG7sZ/NmC7NnJwJgNge4/fYyrruu\nPMhRi+wRskmc260kTkRERET+mskELVv6eeihEnbssHDPPTEA3HBDGZ06eXjllQgeeiia447zcc45\nrv2uX7PGRmysQfPmKpYix05IJnF+P3g8SuJERERE5NBYLPDcc0VMnBjDhRfaOP306mqXWVluBg+2\nMHZsLJ99VsXGjVbOOcfFqFEVrF5tY+DARBISDBYvztOadHLMhOScOI+neuyywxHkQERERESk3nA6\nAzzxRDEDB+5JxhwOeOmlQlJTDT78MJy8PDN33RXDpEnRjB4dR3y8wa5dZsaPj1W1SzlmQrInrqqq\n+k/1xImIiIjIkUpJMVi2LA8Aw4Cbb45l+nQnDkeA997L5z//cfDII9E4nQbx8Qbt2nk55xyXKl3K\nUROSSZzbvbsnTkmciIiIiBy53QmZxQLTphXTsqWPk0/20q6dlzZtvHz/vZ0334zAMMDvN9G+vYf7\n7y+hY0dvcAOXkBSSwyldrur/ZeqJExEREZHaZjbD2LHl9OnjBqoTuzlzCtm8OZdNm3J5/PEiduyw\nMHBgIjNmRGqYpdQ6JXEiIiIiIrXEYoEhQ6r47LM8+vRxcf/9MXTtmkznzsn075/IO++EU1pqYscO\nM6WlGm8ph0fDKUVEREREallUVIAXXyziX/9ys2KFg/DwAN99Z+f66+NqznE4AvTvX0WPHi62b7fg\ncAS4+OJK7PYgBi71QkgmceqJExEREZFgM5ngsssqueyySqC6KMqyZQ5++cWK0xngl19svPlmOG+8\nEVFzzbx5kTz1VBEnneQLVthSD4TkcMrdPXFhYUEORERERETkd2YznHWWm6uvrmDkyEoefLCEVat2\nsmjRLtauzWXWrEJ27DBz7rlJLFq050HWMKr/DATg2WedZGamsHChHnQbMvXEiYiIiIgESUxMgHbt\nqitYnn22i4wMD5dfHs/o0XEMHVrJmjV2Nmyw0ru3C7MZFiwIJyHBz+jR8Vx/fRm33FKGNSSf6OWv\nhGRPnMtV/afmxImIiIhIfZKUZPDmmwX06+fi9dcjcToNLrqoki+/tLNgQTi33lrKN9/s5OKLK3j6\n6SgGDUpk40YLOTkWvv3WRmHhwR/vf/nFWvO8LPVTSObt6okTERERkfoqPLy6KEpVVRHh4dX7Jk0q\noaTETFJS9djKRx4pISvLw4QJMZx+eso+1zdu7OOuu0o5//z9M7VVq2z0759EVpabuXMLau4v9YuS\nOBERERGROmjvBMtupyaB223AgCo6dXIzf34ECQkGqal+Nm60smBBOFddFc+XX1Ywbdqe8wMBmDw5\nmqgogy+/tHPllfHMnFmoapj1UEgmcVpiQEREREQagrQ0g3Hjyvfa4+aKKyqYMiWa55938uGHAUaP\ndnLxxRWsWmXnm28cTJlSjMkEt98ey+jR8cyYUaiCgPVMSCZx6okTEZFgW716NbNnz8YwDLKzsxkw\nYMA+xz/55BP+/e9/YzabCQsLY8yYMTRp0iRI0YpIKLHZ4P/bu9fgqOozjuPfveRKQpLdYEIi0RIS\nW2i9xETbKDhoChSpsRSIFMulY3EMDG0cCt4AUbFUTNMRRabKMJRxLKJcai9qAZHBtCUEUWoVhVbL\nyJolWUiAXNjdc/oiw9pIYgKE7Nnl93m1e/Zs8jz8d3ny7Hn2nAULmrj11haeftrN0qX9+fWvk0lM\nNBk8OMAddzQTEwPBIDzwQCpTp7qZN68JpxO2bo1n06YExo1rYf784+FORboQlU1cW5sNm83UoWER\nEQkLwzBYtWoVDz30EG63m/vvv5/CwsIOTdqNN97IqFGjANi9ezdr1qzhwQcfDFfIIhKFCgr8/OEP\nAXbsOMa6dYm88UY8ixc3EhPT/vi0ac3062dSUZFKaekAAGw2k9zcAE89lUxqqsH3vtfKs88mkZ/v\nZ/r0Zmw28PlsxMZCUpIOmIRLVDZxra024uJMbLZwRyIiIhejAwcOkJmZSUZG+8kGiouLqamp6dDE\nJSZ+cXHf1tZWbCpaInKBDB0aYPHiJhYvbjrjsQkTWigoOMV//+ukpcXGlVf6ycwMcs89aTzySAqP\nP94fwwDDsLFzZxzZ2UFeeKEfqakGK1f6KCry4/XaiY01SU1VU9dXoraJ01yviIiEi8/nw+12h+67\n3W4+/lOqgBAAAA/JSURBVPjjM/Z77bXX+NOf/kQgEGDhwoV9GaKISMjgwUEGDw522PbUU0eJizNJ\nSTGYNesEr76awJIl/TFN+MEPWqipiWXChHRycwPs3x9DVlaATZvqyc42+PRTB59/7qCo6BT2qLyg\nWfj1qInrbq4foLq6mvXr12Oz2bjsssv42c9+BkBZWRk5OTkApKenM3/+/F4Mv3Ntbfo+nIiIWN+Y\nMWMYM2YMO3fu5JVXXmH27Nln7LNlyxa2bNkCwNKlS0lPTz+n3+V0Os/5uVajXKxJuVjT+eTy4oun\nb7n41rdg/Hg/8fFw+eVOjh0z+MUvbBw65GDChCDLlzuYOjWDKVOCPPaYg7Y2G4MGmdx1V5Cf/9w4\n7wMsWpMv/YzudujJXL/H42HTpk08+uijJCUl0djYGHosNjaWZcuWnVeQZ6v9SJyaOBERCQ+Xy0VD\nQ0PofkNDAy6Xq8v9i4uLee655zp9rKSkhJKSktD9+vr6c4opPT39nJ9rNcrFmpSLNfVmLqf7jtM/\n7pe//OKxa66JZcoUNwsWOBk1qoVx41p55ZUEFi2KZ/XqAJMmNfPBBzEEAjB9+km++U0/v/99Itu3\nx1NXZycx0aSy8hhf/3rggucRbj3JJSsr6ysf77aJ68lc/9atWxk9ejRJSUkApKSkdBv8hdTWZtPl\nBUREJGxyc3PxeDx4vV5cLhfV1dXMmTOnwz4ej4eBAwcCsGfPntBtEZFI9J3vnOLFFxuor7czdmwr\nNhv88Ict7NgRywMPpPLEE/259NIAra02/vKXBBwOk2DQxrBhfnJzA9TWxlJams7y5UcpKWnrcgzz\n+HEbycn6O7/bJq4nc/2HDx8GYMGCBRiGwcSJE7n66qsB8Pv93HfffTgcDkpLS7nuuut6M/5O6Uic\niIiEk8Ph4Cc/+QlLlizBMAxGjhzJoEGDWLduHbm5uRQWFvLaa6+xb98+HA4HSUlJzJo1K9xhi4ic\nl+uvP3XGthEjTrF9u5emJjsul0FrK7z8ciIHDzqZMKGZYcPaj7x99pmd6dPdzJjhJiXF4Mor/Vxx\nhZ+8vAD5+QH697exeLGLnTvjeP75o4we3drX6VlKr5zYxDAMPB4PixYtwufzsWjRIp588kn69evH\nihUrcLlc1NXV8cgjj5CTk0NmZmaH5/fWvD+0z5gaBiQlEfFzs9Ey+xsteYBysSrlYk3RlMu5KCgo\noKCgoMO2srKy0O0ZM2b0dUgiImHhdILLZQAQHw933tl8xj7Z2QabN9fz6qvx1NbGsm9fDC+8kEhL\nyxeH5FwuOzk5Qe69N5W//tVLVpbRZzlYTbdNXE/m+l0uF3l5eTidTi655BIGDhyIx+NhyJAhoX0z\nMjIYOnQon3zyyRlNXG/N+0N749bUBElJBvX1vnP+OVYQLbO/0ZIHKBerUi7W1Bsz/yIicvFITDQp\nK2uhrKwFAMOAw4cdfPSRE78/hRtuOEJdnZ0xYwYwa1YapaUtnDhh59vfbqOgwN/lCKZpwpEjdtxu\nA4ejfVtzc/vXr07fjzTdNnE9meu/7rrr2LlzJyNHjqSpqQmPx0NGRgYnTpwgLi6OmJgYmpqa2L9/\nP6WlpRcsmdNaW22kp2ucUkREREQkUtntcOmlQS69NEh6ukF9vUlSUpDHH2+koiKVXbviQvtmZwd4\n+OEmxo5t5Y034pg7NxWnE7Kygnz6qQOfz8GAAUFGj27l00+dVFfHEh9vUlDgp7z8BCNGtHX43cFg\n+++36iU8u23iejLXf9VVV/Huu+9SUVGB3W7nzjvvJDk5mf379/Pb3/4Wu92OYRjcfvvtHU6IcqG0\ntUFcXPf7iYiIiIhIZJk4sYXhw9tPfuJ0mmzbFs/zz/fjpz91MXx4Gzt3xjJsmJ9hwwJ89pmDUaNa\nyc8PsHt3LOvXJ5KVFWTmzJM0N9t48804pk51sXLlUcaMaf+e3e7dMcyenUZyssmvfnWMggJ/mDM+\nU4++E9fdXL/NZmPatGlMmzatwz5XXHEFlZWVvRDm2dGJTUREREREoldm5hffh5swoYXbbmvhiSf6\n8+yzSZSWNlNZeYyEhI7Pufvuk2ccYWtqsjFlipu7707jtttacDhgw4YEsrKC+Hx2brstnVtvbWXs\n2BYuuyyI12vn8suD5Od3fimEvtIrJzaxGl1iQERERETk4hEbCw891MQ995zA5TK6HIP88nfg+vc3\nefHFBioqUvnHP2I5edLO97/fwuOPN2K3Q1VVMi+/nMAf//hFR2izmZSVNTNv3nEyMtqbyUAAPB4H\n/foZJCebxMRcqEzbRWUTpyNxIiIiIiIXH7f77M9YmZRk8txzRzt9bOHCJh58sIk9e2I4erT95Ch/\n/nMCq1b1Y8uWeF54oYHMTIMf/9jFe+/FApCWFuSf/6w7rzy6E5VNXFubmjgRERERETl/DgcUFX3x\nvbhrr/VTVtbMlCkuJk5MJy3NwOu1s2BBIzExfXMylKhr4oJB8PvVxImIiIiIyIWRnx9g06YGJk92\n0dDgYN26Bq69tu9OgBJ1TZzNBm+84T2nQ6kiIiIiIiI9kZ0d5PXXj9DWZiM1tW8PIEVdE2e3w7Bh\n4T1bjIiIiIiIRL+EBEhI6PsJwC6uay4iIiIiIiJWpCZOREREREQkgqiJExERERERiSBq4kRERERE\nRCKImjgREREREZEIoiZOREREREQkgqiJExERERERiSBq4kRERERERCKImjgREREREZEIoiZORERE\nREQkgthM0zTDHYSIiIiIiIj0TNQdibvvvvvCHUKviZZcoiUPUC5WpVysKZpyiQbRtB7KxZqUizVF\nSy7Rkgf0Ti5R18SJiIiIiIhEMzVxIiIiIiIiEcTx8MMPPxzuIHrb4MGDwx1Cr4mWXKIlD1AuVqVc\nrCmacokG0bQeysWalIs1RUsu0ZIHnH8uOrGJiIiIiIhIBNE4pYiIiIiISARxhjuA3rJ3715Wr16N\nYRjccsst3H777eEOqcfq6+t55plnOHbsGDabjZKSEsaOHctLL73E1q1b6d+/PwCTJ0+moKAgzNF2\nb9asWcTHx2O323E4HCxdupQTJ05QVVXFkSNHGDBgABUVFSQlJYU71K90+PBhqqqqQve9Xi+TJk3i\n5MmTEbEuK1asYM+ePaSkpFBZWQnQ5TqYpsnq1at55513iIuLo7y83FIjC53lsnbtWmpra3E6nWRk\nZFBeXk6/fv3wer1UVFSQlZUFQF5eHjNnzgxn+B10lstXvdc3btzItm3bsNvtzJgxg6uvvjpssf+/\nzvKoqqri8OHDADQ3N5OYmMiyZcssvyYXg0itkaqP1qT6qPp4IURLfYQ+qpFmFAgGg+bs2bPNzz//\n3PT7/ebcuXPNQ4cOhTusHvP5fObBgwdN0zTN5uZmc86cOeahQ4fMdevWmZs3bw5zdGevvLzcbGxs\n7LBt7dq15saNG03TNM2NGzeaa9euDUdo5ywYDJp33XWX6fV6I2Zd3n//ffPgwYPmvffeG9rW1TrU\n1taaS5YsMQ3DMPfv32/ef//9YYm5K53lsnfvXjMQCJim2Z7X6Vzq6uo67Gc1neXS1Wvq0KFD5ty5\nc81Tp06ZdXV15uzZs81gMNiX4Xapszz+35o1a8z169ebpmn9NYl2kVwjVR+tT/UxvFQfrVcfTbNv\namRUjFMeOHCAzMxMMjIycDqdFBcXU1NTE+6weiwtLS30qU5CQgLZ2dn4fL4wR9W7ampquOmmmwC4\n6aabImp9APbt20dmZiYDBgwIdyg9NnTo0DM+ze1qHXbv3s2IESOw2Wzk5+dz8uRJjh492ucxd6Wz\nXK666iocDgcA+fn5EfOe6SyXrtTU1FBcXExMTAyXXHIJmZmZHDhw4AJH2DNflYdpmvztb3/jhhtu\n6OOopDORXCNVH61P9TG8VB+tVx+hb2pkVIxT+nw+3G536L7b7ebjjz8OY0Tnzuv18p///IchQ4bw\n4Ycf8vrrr7Njxw4GDx7M1KlTLT9icdqSJUsA+O53v0tJSQmNjY2kpaUBkJqaSmNjYzjDO2tvv/12\nhzdbpK5LV+vg8/lIT08P7ed2u/H5fKF9rW7btm0UFxeH7nu9XubNm0dCQgJ33HEH3/jGN8IYXc90\n9pry+Xzk5eWF9nG5XBFRjD/44ANSUlIYOHBgaFskrkm0iJYaqfpoTaqP1qb6aD29VSOjoomLFq2t\nrVRWVjJ9+nQSExMZNWoUEyZMAGDdunX87ne/o7y8PMxRdu/RRx/F5XLR2NjIY489FprxPc1ms2Gz\n2cIU3dkLBALU1tbyox/9CCBi1+XLIm0durJhwwYcDgfDhw8H2j+5X7FiBcnJyfz73/9m2bJlVFZW\nkpiYGOZIuxYtr6nTvvxHXSSuiViL6qM1qT5am+qjNfVWjYyKcUqXy0VDQ0PofkNDAy6XK4wRnb1A\nIEBlZSXDhw/n+uuvB9o/CbLb7djtdm655RYOHjwY5ih75vS/fUpKCkVFRRw4cICUlJTQ+MHRo0dD\nX1CNBO+88w5f+9rXSE1NBSJ3XYAu18HlclFfXx/aL1LeQ9u3b6e2tpY5c+aECm5MTAzJyclA+zVY\nMjIy8Hg84QyzW129pr78f5vP57P8ugSDQXbt2tXhk99IXJNoEuk1UvXRulQfrUv10Zp6s0ZGRROX\nm5uLx+PB6/USCASorq6msLAw3GH1mGmarFy5kuzsbMaNGxfa/v8z17t27WLQoEHhCO+stLa20tLS\nErr93nvvkZOTQ2FhIW+99RYAb731FkVFReEM86x8+ROTSFyX07pah8LCQnbs2IFpmnz00UckJiZa\nflRk7969bN68mfnz5xMXFxfa3tTUhGEYANTV1eHxeMjIyAhXmD3S1WuqsLCQ6upq/H4/Xq8Xj8fD\nkCFDwhVmj+zbt4+srKwO43uRuCbRJJJrpOqjtak+WpPqo3X1Zo2Mmot979mzhzVr1mAYBiNHjmT8\n+PHhDqnHPvzwQxYuXEhOTk7o05LJkyfz9ttv88knn2Cz2RgwYAAzZ860/H8cdXV1PPnkk0D7pw03\n3ngj48eP5/jx41RVVVFfXx8xp1CG9kJbXl7O008/HTqsvXz58ohYl9/85jf861//4vjx46SkpDBp\n0iSKioo6XQfTNFm1ahXvvvsusbGxlJeXk5ubG+4UQjrLZePGjQQCgdDr6PQpef/+97/z0ksv4XA4\nsNvtTJw40VJ/sHaWy/vvv9/la2rDhg28+eab2O12pk+fzjXXXBPmDNp1lsfNN9/MM888Q15eHqNG\njQrta/U1uRhEao1UfbQu1UdrUH20Xn2EvqmRUdPEiYiIiIiIXAyiYpxSRERERETkYqEmTkRERERE\nJIKoiRMREREREYkgauJEREREREQiiJo4ERERERGRCKImTkREREREJIKoiRMREREREYkgauJERERE\nREQiyP8AgsKcUdBCohIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 1080x360 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aKMA0luELvtI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "check_df = pd.DataFrame(list(zip(df['review'].values, df['cleaned'].values, df['sentiment'].values, y_pred)), columns = ['review','cleaned','sentiment','predict'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JrSVoLkISSbm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "outputId": "9f3b9b01-1e9a-4dcc-e133-22343632ab8d"
      },
      "source": [
        "check_df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>review</th>\n",
              "      <th>cleaned</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>predict</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>This movie is the beginning of the culmination...</td>\n",
              "      <td>beginning culmination masterfully woven cinema...</td>\n",
              "      <td>1</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Over the past decade, Marvel has earned itself...</td>\n",
              "      <td>past decade earned benefit doubt consistently ...</td>\n",
              "      <td>1</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>This film is way better than endgame!\\nThe act...</td>\n",
              "      <td>way better action better writing better dialog...</td>\n",
              "      <td>1</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Summer movies often hype themselves as spectac...</td>\n",
              "      <td>summer often hype spectacular event missed ad ...</td>\n",
              "      <td>1</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I was amazed to see so many negative reviews; ...</td>\n",
              "      <td>amazed negative impossible please hour long co...</td>\n",
              "      <td>1</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5022</th>\n",
              "      <td>Admittingly, I was not a fan of the original. ...</td>\n",
              "      <td>admittingly fan original found first eye candi...</td>\n",
              "      <td>0</td>\n",
              "      <td>[False]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5023</th>\n",
              "      <td>Like the first round, it appears it is always ...</td>\n",
              "      <td>like first round appears always smart keep ori...</td>\n",
              "      <td>1</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5024</th>\n",
              "      <td>With the exception of 'Captain America: The Wi...</td>\n",
              "      <td>exception america winter soldier cinematic uni...</td>\n",
              "      <td>1</td>\n",
              "      <td>[False]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5025</th>\n",
              "      <td>I loved it! It was funny and witty and had all...</td>\n",
              "      <td>loved funny witty ingredient make best film fa...</td>\n",
              "      <td>1</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5026</th>\n",
              "      <td>There are a few minor spoilers. I will try to ...</td>\n",
              "      <td>minor spoiler try got first weekend got fun sp...</td>\n",
              "      <td>1</td>\n",
              "      <td>[False]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5027 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                 review  ...  predict\n",
              "0     This movie is the beginning of the culmination...  ...   [True]\n",
              "1     Over the past decade, Marvel has earned itself...  ...   [True]\n",
              "2     This film is way better than endgame!\\nThe act...  ...   [True]\n",
              "3     Summer movies often hype themselves as spectac...  ...   [True]\n",
              "4     I was amazed to see so many negative reviews; ...  ...   [True]\n",
              "...                                                 ...  ...      ...\n",
              "5022  Admittingly, I was not a fan of the original. ...  ...  [False]\n",
              "5023  Like the first round, it appears it is always ...  ...   [True]\n",
              "5024  With the exception of 'Captain America: The Wi...  ...  [False]\n",
              "5025  I loved it! It was funny and witty and had all...  ...   [True]\n",
              "5026  There are a few minor spoilers. I will try to ...  ...  [False]\n",
              "\n",
              "[5027 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iNpLlo21vlwX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "check_df.to_csv('check_dlstm.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uvFNRqJf2_ph",
        "colab_type": "text"
      },
      "source": [
        "# Predict"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wE6l-_ye928X",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "nan = pd.read_csv(\"nan.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "46sUeQBIGzCQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "ee2c70fe-a176-4b3b-ac7b-1ea9d0b8bf32"
      },
      "source": [
        "nan.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>rating</th>\n",
              "      <th>title</th>\n",
              "      <th>review</th>\n",
              "      <th>number</th>\n",
              "      <th>cleaned</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>69</td>\n",
              "      <td>NaN</td>\n",
              "      <td>A tedious and dependent film</td>\n",
              "      <td>SPOILER: The plot is simple that the supervill...</td>\n",
              "      <td>0</td>\n",
              "      <td>spoiler simple supervillian succeeds russo bro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>126</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Thanos the Emo Crybaby...Thanos Mickey Mouse S...</td>\n",
              "      <td>12 jokes within 3 minutes in conversation betw...</td>\n",
              "      <td>0</td>\n",
              "      <td>joke within minute conversation doctor strange...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>139</td>\n",
              "      <td>NaN</td>\n",
              "      <td>A snore -- literally</td>\n",
              "      <td>This film was complete and utter tosh. Judging...</td>\n",
              "      <td>0</td>\n",
              "      <td>complete utter tosh judging way positively rev...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>146</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Thanos and Bubbles, Kiddie Mickey Mouse Style</td>\n",
              "      <td>I don't recognize the THANOS The MAD TITAN fro...</td>\n",
              "      <td>0</td>\n",
              "      <td>recognize thanos mad titan comic book thanos w...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>176</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Good movie, but completely ruined</td>\n",
              "      <td>Spoilers!!!Spoilers: the movie was pretty good...</td>\n",
              "      <td>0</td>\n",
              "      <td>spoiler spoiler pretty good bit quippy good en...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0  ...                                            cleaned\n",
              "0          69  ...  spoiler simple supervillian succeeds russo bro...\n",
              "1         126  ...  joke within minute conversation doctor strange...\n",
              "2         139  ...  complete utter tosh judging way positively rev...\n",
              "3         146  ...  recognize thanos mad titan comic book thanos w...\n",
              "4         176  ...  spoiler spoiler pretty good bit quippy good en...\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UqUZKx2pHjb8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ab348ea0-5e2c-4d84-8243-da6adcc2453c"
      },
      "source": [
        "nan.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(533, 6)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oZmDfVllGvgy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for sent in nan:\n",
        "    review = nan['review'].values\n",
        "    cleaned = nan['cleaned'].values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hQDuXnbdG_bk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cleaned_str = cleaned.astype(str)\n",
        "\n",
        "tokenizer = Tokenizer(char_level=False)\n",
        "tokenizer.fit_on_texts(cleaned_str)\n",
        "cleaned_bow = tokenizer.texts_to_sequences(cleaned_str) #text_train"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q5sbZADIHAYc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "1f4f1315-e842-4c02-d265-ca83cd440d5c"
      },
      "source": [
        "x_pred = pad_sequences(cleaned_bow, maxlen=max_len)\n",
        "print('Shape of data tensor:', x_pred.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape of data tensor: (533, 500)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "tBYTrU1Z3Gee",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "56a1affd-7b72-47d8-89b0-6ba279f6c576"
      },
      "source": [
        "pred = model.predict(x_pred, batch_size=100, verbose = 1)\n",
        "y_pred = (pred > 0.5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "533/533 [==============================] - 2s 4ms/sample\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W4C7LWibHHRQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred_df = pd.DataFrame(list(zip(review, y_pred)), columns = ['review','predict'])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "prz4vzbfHxgE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "outputId": "93aeb354-9983-4c10-ad3b-a7b86efef8b0"
      },
      "source": [
        "pred_df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>review</th>\n",
              "      <th>predict</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>SPOILER: The plot is simple that the supervill...</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>12 jokes within 3 minutes in conversation betw...</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>This film was complete and utter tosh. Judging...</td>\n",
              "      <td>[False]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>I don't recognize the THANOS The MAD TITAN fro...</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Spoilers!!!Spoilers: the movie was pretty good...</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>528</th>\n",
              "      <td>Just finished watching this movie. My husband ...</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>529</th>\n",
              "      <td>Ok at best. Big script let down from what's su...</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>530</th>\n",
              "      <td>Had this movie come out in 2009 close to \"Iron...</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>531</th>\n",
              "      <td>I'm a middle-aged white male. Saw Captain Marv...</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>532</th>\n",
              "      <td>PROS:Brie Larson's Is Great As Captain MarvelS...</td>\n",
              "      <td>[True]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>533 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                review  predict\n",
              "0    SPOILER: The plot is simple that the supervill...   [True]\n",
              "1    12 jokes within 3 minutes in conversation betw...   [True]\n",
              "2    This film was complete and utter tosh. Judging...  [False]\n",
              "3    I don't recognize the THANOS The MAD TITAN fro...   [True]\n",
              "4    Spoilers!!!Spoilers: the movie was pretty good...   [True]\n",
              "..                                                 ...      ...\n",
              "528  Just finished watching this movie. My husband ...   [True]\n",
              "529  Ok at best. Big script let down from what's su...   [True]\n",
              "530  Had this movie come out in 2009 close to \"Iron...   [True]\n",
              "531  I'm a middle-aged white male. Saw Captain Marv...   [True]\n",
              "532  PROS:Brie Larson's Is Great As Captain MarvelS...   [True]\n",
              "\n",
              "[533 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XzteJqDNHyPk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred_df.to_csv('pred_dlstm.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f3IJpIgSvefv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "acfd7d16-d909-4620-816e-85bd8bb40d85"
      },
      "source": [
        "model.save(\"dlstm.h5\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:TensorFlow optimizers do not make it possible to access optimizer attributes or optimizer state after instantiation. As a result, we cannot save the optimizer as part of the model save file.You will have to compile your model again after loading it. Prefer using a Keras optimizer instead (see keras.io/optimizers).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dMw2C2U3voMV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}