Network_Intrusion_Detector.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mini-Project 2: Network Intrusion Detector\n",
    "\n",
    "\n",
    "#### CSC 180  Intelligent Systems (Fall 2019)\n",
    "#### Derrek Gass , Alexander Lee, Jimmy Le\n",
    "#### 10-11-2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import figure, show\n",
    "\n",
    "import collections\n",
    "\n",
    "import io\n",
    "import requests\n",
    "import shutil\n",
    "import os, json\n",
    "import csv\n",
    "\n",
    "from sklearn import metrics\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "import sklearn.feature_extraction.text as sk_text\n",
    "from sklearn.metrics import confusion_matrix, classification_report, roc_curve, auc\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Activation, Dropout, Flatten, Conv2D, MaxPooling2D\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from tensorflow.keras import optimizers, regularizers\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Encode text values to dummy variables(i.e. [1,0,0],[0,1,0],[0,0,1] for red,green,blue)\n",
    "def encode_text_dummy(df, name):\n",
    "    dummies = pd.get_dummies(df[name])\n",
    "    for x in dummies.columns:\n",
    "        dummy_name = \"{}-{}\".format(name, x)\n",
    "        df[dummy_name] = dummies[x]\n",
    "    df.drop(name, axis=1, inplace=True)\n",
    "\n",
    "\n",
    "# Encode text values to indexes(i.e. [1],[2],[3] for red,green,blue).\n",
    "def encode_text_index(df, name):\n",
    "    le = preprocessing.LabelEncoder()\n",
    "    df[name] = le.fit_transform(df[name])\n",
    "    return le.classes_\n",
    "\n",
    "\n",
    "# Encode a numeric column as zscores\n",
    "def encode_numeric_zscore(df, name, mean=None, sd=None):\n",
    "    if mean is None:\n",
    "        mean = df[name].mean()\n",
    "\n",
    "    if sd is None:\n",
    "        sd = df[name].std()\n",
    "\n",
    "    df[name] = (df[name] - mean) / sd\n",
    "\n",
    "\n",
    "# Convert all missing values in the specified column to the median\n",
    "def missing_median(df, name):\n",
    "    med = df[name].median()\n",
    "    df[name] = df[name].fillna(med)\n",
    "\n",
    "\n",
    "# Convert all missing values in the specified column to the default\n",
    "def missing_default(df, name, default_value):\n",
    "    df[name] = df[name].fillna(default_value)\n",
    "\n",
    "\n",
    "# Convert a Pandas dataframe to the x,y inputs that TensorFlow needs\n",
    "def to_xy(df, target):\n",
    "    result = []\n",
    "    for x in df.columns:\n",
    "        if x != target:\n",
    "            result.append(x)\n",
    "    # find out the type of the target column. \n",
    "    target_type = df[target].dtypes\n",
    "    target_type = target_type[0] if isinstance(target_type, collections.Sequence) else target_type\n",
    "    # Encode to int for classification, float otherwise. TensorFlow likes 32 bits.\n",
    "    if target_type in (np.int64, np.int32):\n",
    "        # Classification\n",
    "        dummies = pd.get_dummies(df[target])\n",
    "        return df[result].values.astype(np.float32), dummies.values.astype(np.float32)\n",
    "    else:\n",
    "        # Regression\n",
    "        return df[result].values.astype(np.float32), df[target].values.astype(np.float32)\n",
    "\n",
    "# Nicely formatted time string\n",
    "def hms_string(sec_elapsed):\n",
    "    h = int(sec_elapsed / (60 * 60))\n",
    "    m = int((sec_elapsed % (60 * 60)) / 60)\n",
    "    s = sec_elapsed % 60\n",
    "    return \"{}:{:>02}:{:>05.2f}\".format(h, m, s)\n",
    "\n",
    "\n",
    "# Remove all rows where the specified column is +/- sd standard deviations\n",
    "def remove_outliers(df, name, sd):\n",
    "    drop_rows = df.index[(np.abs(df[name] - df[name].mean()) >= (sd * df[name].std()))]\n",
    "    df.drop(drop_rows, axis=0, inplace=True)\n",
    "\n",
    "\n",
    "# Encode a column to a range between normalized_low and normalized_high.\n",
    "def encode_numeric_range(df, name, normalized_low=-1, normalized_high=1,\n",
    "                         data_low=None, data_high=None):\n",
    "    if data_low is None:\n",
    "        data_low = min(df[name])\n",
    "        data_high = max(df[name])\n",
    "\n",
    "    df[name] = ((df[name] - data_low) / (data_high - data_low)) \\\n",
    "               * (normalized_high - normalized_low) + normalized_low\n",
    "\n",
    "    \n",
    "# Regression chart.\n",
    "def chart_regression(pred,y,sort=True):\n",
    "    t = pd.DataFrame({'pred' : pred, 'y' : y.flatten()})\n",
    "    if sort:\n",
    "        t.sort_values(by=['y'],inplace=True)\n",
    "    a = plt.plot(t['y'].tolist(),label='expected')\n",
    "    b = plt.plot(t['pred'].tolist(),label='prediction')\n",
    "    plt.ylabel('output')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "# Plot a confusion matrix.\n",
    "# cm is the confusion matrix, names are the names of the classes.\n",
    "def plot_confusion_matrix(cm, names, title='Confusion matrix', cmap=plt.cm.Blues):\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(names))\n",
    "    plt.xticks(tick_marks, names, rotation=45)\n",
    "    plt.yticks(tick_marks, names)\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    \n",
    "\n",
    "# Plot an ROC. pred - the predictions, y - the expected output.\n",
    "def plot_roc(pred,y):\n",
    "    fpr, tpr, thresholds = roc_curve(y, pred)\n",
    "    roc_auc = auc(fpr, tpr)\n",
    "\n",
    "    plt.figure()\n",
    "    plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % roc_auc)\n",
    "    plt.plot([0, 1], [0, 1], 'k--')\n",
    "    plt.xlim([0.0, 1.0])\n",
    "    plt.ylim([0.0, 1.05])\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.ylabel('True Positive Rate')\n",
    "    plt.title('Receiver Operating Characteristic (ROC)')\n",
    "    plt.legend(loc=\"lower right\")\n",
    "    plt.show()\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#encode_csv = X_test_prep.to_csv(r'data/nid_test_prep.csv', header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_prep = pd.read_csv(r'data/nid_test_prep.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "y=X_test_prep['outcome_normal.']\n",
    "X_test_prep=X_test_prep.drop('outcome_normal.', axis=1)\n",
    "X_numpy = X_test_prep.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = X_numpy.reshape((len(X_test_prep), 1, len(X_test_prep.columns), 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split into train/test\n",
    "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25, random_state=42)\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "y_train = tf.keras.utils.to_categorical(y_train, 2)\n",
    "y_test = tf.keras.utils.to_categorical(y_test, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(86631, 1, 118, 1)"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(86631, 2)"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28877, 1, 118, 1)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28877, 2)"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convolutional Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define a CNN\n",
    "\n",
    "cnn = Sequential()\n",
    "cnn.add(Conv2D(64, kernel_size=(1, y_train.shape[1]), strides=(1, 1),\n",
    "                 activation='tanh',\n",
    "                 input_shape=(1, x_train.shape[2], 1)))\n",
    "cnn.add(MaxPooling2D(pool_size=(1,2)))\n",
    "cnn.add(Conv2D(x_train.shape[2], kernel_size=(1, y_train.shape[1]), strides=(1, 1),\n",
    "                 activation='tanh'))\n",
    "cnn.add(MaxPooling2D(pool_size=(1,2)))\n",
    "    \n",
    "    \n",
    "cnn.add(Flatten())\n",
    "cnn.add(Dense(32, activation=\"relu\"))\n",
    "cnn.add(Dropout(0.5))\n",
    "cnn.add(Dense(y_train.shape[1], activation=\"softmax\"))\n",
    "\n",
    "cnn.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_20\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_10 (Conv2D)           (None, 1, 117, 64)        192       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_10 (MaxPooling (None, 1, 58, 64)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_11 (Conv2D)           (None, 1, 57, 118)        15222     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_11 (MaxPooling (None, 1, 28, 118)        0         \n",
      "_________________________________________________________________\n",
      "flatten_5 (Flatten)          (None, 3304)              0         \n",
      "_________________________________________________________________\n",
      "dense_43 (Dense)             (None, 32)                105760    \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_44 (Dense)             (None, 2)                 66        \n",
      "=================================================================\n",
      "Total params: 121,240\n",
      "Trainable params: 121,240\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "cnn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 86631 samples, validate on 28877 samples\n",
      "Epoch 1/100\n",
      "86631/86631 - 6s - loss: 0.0143 - acc: 0.9952 - val_loss: 0.0096 - val_acc: 0.9959\n",
      "Epoch 2/100\n",
      "86631/86631 - 5s - loss: 0.0126 - acc: 0.9954 - val_loss: 0.0091 - val_acc: 0.9964\n",
      "Epoch 3/100\n",
      "86631/86631 - 5s - loss: 0.0118 - acc: 0.9958 - val_loss: 0.0089 - val_acc: 0.9966\n",
      "Epoch 4/100\n",
      "86631/86631 - 5s - loss: 0.0111 - acc: 0.9963 - val_loss: 0.0076 - val_acc: 0.9977\n",
      "Epoch 5/100\n",
      "86631/86631 - 5s - loss: 0.0101 - acc: 0.9969 - val_loss: 0.0069 - val_acc: 0.9982\n",
      "Epoch 6/100\n",
      "86631/86631 - 5s - loss: 0.0096 - acc: 0.9970 - val_loss: 0.0062 - val_acc: 0.9978\n",
      "Epoch 7/100\n",
      "86631/86631 - 5s - loss: 0.0090 - acc: 0.9973 - val_loss: 0.0063 - val_acc: 0.9978\n",
      "Epoch 8/100\n",
      "86631/86631 - 5s - loss: 0.0088 - acc: 0.9974 - val_loss: 0.0061 - val_acc: 0.9979\n",
      "Epoch 9/100\n",
      "86631/86631 - 5s - loss: 0.0084 - acc: 0.9973 - val_loss: 0.0059 - val_acc: 0.9981\n",
      "Epoch 10/100\n",
      "86631/86631 - 5s - loss: 0.0083 - acc: 0.9975 - val_loss: 0.0072 - val_acc: 0.9974\n",
      "Epoch 11/100\n",
      "86631/86631 - 5s - loss: 0.0080 - acc: 0.9974 - val_loss: 0.0068 - val_acc: 0.9982\n",
      "Epoch 12/100\n",
      "86631/86631 - 5s - loss: 0.0076 - acc: 0.9976 - val_loss: 0.0059 - val_acc: 0.9985\n",
      "Epoch 00012: early stopping\n",
      "Model: CNN, Activation: tanh, tanh, relu, softmax, Optimizer: adam, Kernel number/size: 118/64, 1x1\n",
      "Elapsed time: 0:01:04.74\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "# 1% of dataset\n",
    "# Define batch_size and # of epochs\n",
    "batch_size = 128\n",
    "\n",
    "monitor = EarlyStopping(monitor='val_loss', min_delta=1e-6, patience=3, verbose=1, mode='auto')\n",
    "checkpointer = ModelCheckpoint(filepath=\"data/best_weights.hdf5\", verbose=0, save_best_only=True) # save best model\n",
    "\n",
    "cnn.fit(x_train, y_train,     \n",
    "        batch_size=batch_size,\n",
    "        callbacks=[monitor, checkpointer],\n",
    "        epochs=100,\n",
    "        verbose=2,\n",
    "        validation_data=(x_test, y_test))\n",
    "\n",
    "cnn.load_weights('data/best_weights.hdf5')\n",
    "elapsed_time = time.time() - start_time\n",
    "\n",
    "print(\"Model: CNN, Activation: tanh, tanh, relu, softmax, Optimizer: adam, Kernel number/size: 118/64, 1x1\")\n",
    "print(\"Elapsed time: {}\".format(hms_string(elapsed_time)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100/100 - 0s - loss: 0.0212 - acc: 0.9900\n"
     ]
    }
   ],
   "source": [
    "# evaluate() computes the loss and accuracy\n",
    "score = cnn.evaluate(x_test[0:100], y_test[0:100], verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.021198396283434704\n",
      "Test accuracy: 0.9900000095367432\n"
     ]
    }
   ],
   "source": [
    "print('Test loss: {}'.format(score[0]))\n",
    "print('Test accuracy: {}'.format(score[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.99\n",
      "Averaged F1: 0.9899867765232429\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.98      0.99        44\n",
      "           1       0.98      1.00      0.99        56\n",
      "\n",
      "    accuracy                           0.99       100\n",
      "   macro avg       0.99      0.99      0.99       100\n",
      "weighted avg       0.99      0.99      0.99       100\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_true = np.argmax(y_test[0:100],axis=1)\n",
    "pred = cnn.predict(x_test[0:100])\n",
    "pred = np.argmax(pred,axis=1)\n",
    "\n",
    "\n",
    "score = metrics.accuracy_score(y_true, pred)\n",
    "print('Accuracy: {}'.format(score))\n",
    "\n",
    "\n",
    "f1 = metrics.f1_score(y_true, pred, average='weighted')\n",
    "print('Averaged F1: {}'.format(f1))\n",
    "\n",
    "           \n",
    "print(metrics.classification_report(y_true, pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZxN9f/A8dfbviZMi1Bjy4x9mWRLSpYkJIWIREK2SCotSkoS8sVXfpJKRSpbWlCWb0Ua2XepGEuWGDtjvH9/nDPTNWa5JnfOnZn38/G4j7lnf9/PnHve93w+53yOqCrGGGNMUrJ4HYAxxpjgZonCGGNMsixRGGOMSZYlCmOMMcmyRGGMMSZZliiMMcYkyxJFBici7UVkgddxBBMROSEiJT3YbqiIqIhkS+ttB4KIbBSR+qlYLtX7pIg0EpHZqVk2tUQkp4hsEZFr03K7wcQSRRoSkT9E5LR7oNovIlNFJF8gt6mqH6lqo0Buw5eI1BaR70XkuIhEi8g8ESmXVttPJJ4lItLVd5yq5lPVnQHa3s0iMlNEDrmff52I9BeRrIHYXmq5Cav0v1mHqpZX1SUpbOeS5Pgv98nXgOE+61cROel+p/aIyKiEZS0izURkpTvfYRH5SESKJZiniIi8KyL73H13i4i8LCJ5VfUsMAUYlMqY0z1LFGnvXlXNB1QBqgLPehxPqiT2q1hEagELgDnADUAJYC3wYyB+wQfbL3MRKQX8DOwGKqpqAeABIALIf4W35dln92rbInILUEBVVySYVNn9Tt0OtAEe9VmmNfAx8DYQApQHzgI/iEhBd55CwHIgN1BLVfMDDYGrgVLuqj4GOolIzgB9vOCmqvZKoxfwB3CXz/AIYL7PcE5gJLAL+AuYCOT2md4CWAMcA34DmrjjCwDvAvuAPcCrQFZ32iPAD+77icDIBDHNAfq7728APgcOAr8DfXzmGwJ8Bkxzt981kc/3P2BCIuO/Bj5w39cHooDngENumbT3pwx8lh0E7Ac+BAoCX7oxH3HfF3PnHwbEAmeAE8A4d7wCpd33U4HxwHzgOM6BvpRPPI2ArUA0MAFYmthnd+ed5vv/TGR6qLvtTu7nOwQM9pleA+eAddT9X44DcvhMV+AJYDvwuzvubZzEdAxYBdzmM39Wt5x/cz/bKqA4sMxd10m3XNq48zfD2b+OAj8BlRLsu4OAdTgH2mz47M9u7JFuHH8Bo9zxu9xtnXBftfDZJ915ygMLgb/dZZ9LovxeBCYnGBf/v3SHPwXGu+8F+BN4OsEyWYANwCvu8KvAeiBLCt/f7cDtXh9HvHh5HkBmeiX4YhVzd863faaPAeYChXB+gc4DXnen1XAPVg3dHb0oEOZOmw28A+QFrgVWAo+70+K/lEA996Ai7nBB4DROgsjiHkheBHIAJYGdQGN33iFADNDSnTd3gs+WB+egfEcin7szsM99Xx84D4zCSQq34xywyvpRBnHLvuEumxsoDNzvbj8/MBOY7bPtJSQ4sHNpovjbLd9swEfAdHdaCM6Br5U7ra9bBkkliv1A52T+/6Hutv/Pjb0yzkE33J1eHajpbisU2Az0SxD3Qrds4pJnB7cMsgED3BhyudMG4uxjZXEOmpWBwgnLwB2uBhwAbsVJMJ1w9tecPvvuGpxEk9tnXNz+vBx42H2fD6iZ4DNn89nWI/yzT+bHSYoDgFzu8K1JlN9MYGAy/8swd11P+gwrUCKRdb0MLHffrwBe9uP7OxefH0+Z6eV5AJnp5X6xTuD8ulPgO+Bqd5rgHDB9f83W4p9fju8AoxNZ53Xuwcb3zKMdsNh97/ulFJxfePXc4ceA7933twK7Eqz7WeA99/0QYFkyn62Y+5nCEpnWBIhx39fHOdjn9Zn+KfCCH2VQHziHeyBMIo4qwBGf4SWknCgm+0xrCmxx33eMO5j4lN/uhOvzmR6De5aXxPRQd9vFfMatBNomMX8/YFaCuO9MYR87glMVA86ZUIsk5kuYKP4LDE0wz1bcX9DuvvtoIvtzXKJYhnPwDUniMyeVKNoBq/38/iwEuifyOY65+40Cn/BPcqvrjrtkfwG6A9vd99sTrjeJ7X8EvOhPrBntZW0Uaa+lOnWg9XF+8YS446/B+VW8SkSOishR4Bt3PDi/5H5LZH03AdmBfT7LvYNzZnERdfb26ThfToCHcHb+uPXcELcOdz3P4SSiOLuT+VxHgAtAkUSmFcGpZomfV1VP+gz/iXNWk1IZABxU1TNxAyKSR0TeEZE/ReQYzgHr6stsPN7v8/4Uzi9i3JjiP7NbflHJrOcwiX9+v7bnNoR/6V7ocAyn4TYkwbIX/Q9EZICIbHYbzo/iVEPGLZPUPpOYm4ABCf7/xXHKINFtJ9AFuBnYIiK/iEgzP7d7OTEeIfG2nmo4ZdgG5wdPXnd83D6X0j7p7/8tP061XKZjicIjqroU59fsSHfUIZxqoPKqerX7KqBOIx04X9JSl66J3ThnFCE+y12lquWT2PQnQGsRuQnnS/W5z3p+91nH1aqaX1Wb+oadzOc5iVP98EAikx/EOXuKU1BE8voM3wjs9aMMEothAE7Vyq2qehVO9Ro4v/6TjdkP+3DOlJwViojvcCIW4VSDpdZ/gS1AGfezPMc/nyNO/OcRkdtw2g0eBAqq6tU41ZNxyyS1zyRmNzAswf8/j6p+kti2E1LV7araDucHyhvAZ+7/OKXyv5wY1+Eko8S2r6r6Kc4++KI7eitOYr9onxSRLDj/p7h9chFwnzs+OeE4F2dkOpYovDUGaCgiVVT1Ak7d9ei467VFpKiINHbnfRfoLCINRCSLOy1MVffhXGn0lohc5U4rJSK3J7ZBVV2N0/A7GfhWVeN+Ia0EjonIIBHJLSJZRaSCe6WJv57BuTKkj4jkF5GCIvIqTvXRywnmfVlEcrgHu2bATD/KIDH5cZLLUffqlZcSTP8Lp70lNeYDFUWkpXulzxPA9cnM/xJQW0TeFJHr3fhLi8g0Ebnaj+3lx6lGOSEiYUAPP+Y/j/P/zCYiLwJX+UyfDAwVkTLiqCQihd1pCcvl/4DuInKrO29eEblHRPy6WktEOojINe7/MG6finVju0DS/4MvgetFpJ849yvkF5Fbk5j3K5w2reQMB7qJyPXuGeBTwPMi8pC7X1+PUy5XAaPdZUa5w++7P6Di9rtRIlIpbhinbSjhFVeZgiUKD6nqQeADnPp5cH4d7gBWuFUPi3B+LaOqK3EahUfj/GpcilNdAE5deg5gE87p+Wckfyr9CXAXziV/cbHEAvfi1PH/jvPrfjJOVYa/n+cHoDFO4+8+nCqlqkBdVd3uM+t+N869OFVf3VV1S0plkIQxOA3Dh3C+xN8kmP42zhnUEREZ6+9ncT/PIZxfoyNwqifK4VzZczaJ+X/DSYqhwEYRicY5Y4vEaZdKyVM41YHHcQ7cM1KY/1ucK8q24ZT1GS6uHhqF0/6zACcBvYtTVuC0Ob3vVjM9qKqROG1W43D+Nztw2hL81QTnM5/AKfO2qnpGVU/hXH32o7utmr4LqepxnAs07sXZL7YDdyS2AVX9FYhOJpGgqutxvhsD3eEZwMPAkzj7yCa3DOqo6mF3nr+B2jhtTD+LyHGcs41otxzA+b+8r849FZlO3NUvxqQJce7knaaqyVXhBCW3aiIK53LexV7HkxmJSCOgp6q2TMNt5sSpcqqnqgfSarvBJKhuWDIm2LjVXj/jVG8NxKn/z5TVD8FAVRfgnCGl5TbP4lx4kmlZ1ZMxyauFc1XOIZzqkZaqetrbkIxJW1b1ZIwxJll2RmGMMSZZ6a6NIiQkRENDQ70Owxhj0pVVq1YdUtVrUp7zUukuUYSGhhIZGel1GMYYk66IyJ+pXdaqnowxxiTLEoUxxphkWaIwxhiTLEsUxhhjkmWJwhhjTLIsURhjjElWwBKFiEwRkQMisiGJ6SIiY0Vkh4isE5FqgYrFGGNM6gXyjGIqTtfDSbkbKOO+uuE8tMUYY0yQCdgNd6q6TERCk5mlBfCB+3CRFSJytYgUcR/Ek6SdB0/S5p3lVzBSY4zJmFSVPWuWsmfN0n+1Hi/vzC7KxQ9ZiXLHXZIoRKQbzlkHOa8vnSbBGWNMenby0D5+nfEW+9b/RIGi/+646WWiSPgsYEji+bqqOgmYBFDopnCd8XitQMZljDHpmqoSERHBsZ1beeutt+jTpw/Zs2dP9fq8TBRRQHGf4WI4j8Y0xhiTCj/99BMVK1Ykf/78TJ48mZCQEIoXL57yginw8vLYuUBH9+qnmkB0Su0TxhhjLnX48GEee+wx6tSpw1tvvQVA1apVr0iSgACeUYjIJ0B9IEREooCXgOwAqjoR+ApoivPw8lNA50DFYowxGZGq8sEHH/DUU09x5MgRBg4cyMCBA6/4dgJ51VO7FKYr8ESgtm+MMRndoEGDePPNN6lduzYTJ06kYsWKAdlOunsehTHGZGanT5/m5MmThISE0KVLF8qUKUOXLl3IkiVwLQnWhYcxxqQT33zzDRUqVODxxx8HoGzZsjz22GMBTRJgicIYY4Le3r17efDBB7n77rvJnj07vXr1StPtW9WTMcYEse+++4777ruPc+fOMXToUAYOHEjOnDnTNAZLFMYYE4RiYmLInj07lStXpmnTprz66quULu1NzxRW9WSMMUHk2LFj9O3bl9tuu43Y2FhCQkKYPn26Z0kCLFEYY0xQUFVmzpxJWFgY//nPf4iIiODs2bNehwVY1ZMxxnju4MGDdOrUia+//pqqVasyZ84cbrnlFq/DimdnFMYY47GrrrqKQ4cOMWbMGFauXBlUSQIsURhjjCeWLVtG48aNOXHiBDlz5mTFihX07duXbNmCr6LHEoUxxqShQ4cO0blzZ26//Xa2bdvGH3/8ARDwm+b+jeCNzBhjMhBVZcqUKZQtW5Zp06bx7LPPsnHjRipUqOB1aCkKvnMcY4zJoKZNm0a5cuWYOHEi5cuX9zocv9kZhTHGBMipU6d4/vnniYqKQkT4/PPPWbp0abpKEmCJwhhjAuKrr76ifPnyDBs2jHnz5gFQsGDBoG6LSEr6i9gYY4JYVFQUrVu35p577iF37twsXbqUHj16eB3Wv2KJwhhjrqBhw4Yxf/58XnvtNdasWUO9evW8DulfE+dBc+lHoZvC9e8/N3sdhjHGxFu5ciW5c+emYsWKHD58mOjoaEqWLOl1WBcRkVWqGpGaZe2MwhhjUik6OponnniCmjVrMnjwYAAKFy4cdEni37JEYYwxl0lVmT59OmFhYUycOJHevXszbdo0r8MKGLuPwhhjLtO0adPo2LEjERERfPnll1SvXt3rkALKEoUxxvjh7Nmz7Ny5k/DwcB588EHOnz9Px44dyZo1q9ehBZxVPRljTAoWL15M5cqVady4MWfPniVnzpx07tw5UyQJsERhjDFJOnDgAB07duTOO+8kJiaGSZMmpfnzqoOBVT0ZY0widuzYQY0aNThx4gSDBw9m8ODB5M6d2+uwPGGJwhhjfBw7doyrrrqKUqVK0aVLFx599FHCw8O9DstTVvVkjDHAyZMnGTRoEKGhofGd+L355puZPkmAnVEYYwzz5s2jV69e7Nq1iy5dupAnTx6vQwoqliiMMZnW+fPnefDBB5k1axbly5fnf//7H3Xr1vU6rKBjVU/GmEwnro+7bNmyUaRIEYYPH86vv/5qSSIJliiMMZnKihUriIiI4NdffwVg/PjxDBo0iBw5cngcWfCyRGGMyRSOHDlCjx49qF27Nn/99RdHjhzxOqR0I6CJQkSaiMhWEdkhIs8kMv1GEVksIqtFZJ2INA1kPMaYzGnGjBmEhYUxadIk+vXrx+bNm2nQoIHXYaUbAWvMFpGswHigIRAF/CIic1V1k89szwOfqup/RaQc8BUQGqiYjDGZ05YtWwgNDeWbb76hatWqXoeT7gTyjKIGsENVd6rqOWA60CLBPApc5b4vAOwNYDzGmEzizJkzvPzyy/HPqn7uuef46aefLEmkUiATRVFgt89wlDvO1xCgg4hE4ZxN9E5sRSLSTUQiRSQyJiYmELEaYzKIRYsWUalSJYYMGcLSpUsByJ49e6bpwC8QApkoJJFxCZ+72g6YqqrFgKbAhyJySUyqOklVI1Q1Inv27AEI1RiT3v3111+0b9+ehg0boqosWLCAkSNHeh1WhhDIRBEFFPcZLsalVUtdgE8BVHU5kAsICWBMxpgMauHChXz22We8+OKLrF+/noYNG3odUoYRyDuzfwHKiEgJYA/QFngowTy7gAbAVBEJx0kUBwMYkzEmA1m7di3bt2+ndevWtG/fnjp16lCiRAmvw8pwAnZGoarngV7At8BmnKubNorIKyLS3J1tAPCYiKwFPgEe0bhbJo0xJgknTpxgwIABVK9enWeeeYbz588jIpYkAkTS23G50E3h+vefm70OwxjjkdmzZ9O7d2+ioqLo1q0br7/+OoUKFfI6rKAnIqtUNSI1y1qngMaYdGP9+vXcd999VKxYkRkzZlC7dm2vQ8oUrAsPY0xQi4mJ4fvvvwegYsWKzJ8/n1WrVlmSSEOWKIwxQeunn36ievXqNGzYkB07dgDQtGlT7DL5tGWJwhgTdP7++2+6detGnTp1OHr0KF988QWlS5f2OqxMy9oojDFB5cyZM1SpUoW9e/cyYMAAhgwZQr58+bwOK1OzRGGMCQpRUVEUK1aMXLlyMXToUKpUqULlypW9DstgVU/GGI+dPn2aF198kVKlSsV34tepUydLEkHErzMKEckB3KiqOwIcjzEmE1mwYAE9e/bkt99+o0OHDtSoUcPrkEwiUjyjEJF7gPXAQne4iojMCnRgxpiMrXfv3jRu3JgsWbKwaNEiPvzwQ6677jqvwzKJ8OeM4hXgVmAxgKquERG7/MAYc9liY2MByJo1KzVr1iQkJIRBgwaRK1cujyMzyfGnjSJGVY8mGJe++v0wxnju119/pVatWkyYMAGA9u3b89JLL1mSSAf8SRSbReRBIIuIlBCRMcCKAMdljMkgjh8/zpNPPsktt9zCrl27KFKkiNchmcvkT6LoBVQHLgBfAGeAvoEMyhiTMSxYsIDw8HDefvttHn/8cbZs2ULr1q29DstcJn/aKBqr6iBgUNwIEWmFkzSMMSZJOXLk4Nprr+Xzzz/n1ltv9Tock0opdjMuIr+qarUE41apavWARpYE62bcmOAVExPDqFGjOHbsGMOGDQPgwoULZMlit2x5LSDdjItIY6AJUFRERvlMugqnGsoYY+L98MMPdO/enY0bN/LAAw/EJwhLEulfcv/BA8AGnDaJjT6vBcDdgQ/NGJMeHD58mK5du3Lbbbdx/Phx5s2bx6effmoJIgNJ8oxCVVcDq0XkI1U9k4YxGWPSkcOHDzN9+nSefvppXnzxRfLmzet1SOYK86cxu6iIDAPKAfEXPKvqzQGLyhgT1DZv3synn37KSy+9xM0338yuXbvscaQZmD/nhlOB9wDBqXL6FJgewJiMMUHq1KlTDB48mMqVK/P2228TFRUFYEkig/MnUeRR1W8BVPU3VX0euCOwYRljgs0333xDhQoVeO2113jooYfYunUrxYoV8zoskwb8qXo6KyIC/CYi3YE9wLWBDcsYE0xOnDjBww8/TOHChVm8eDH169f3OiSThvw5o3gSyAf0AeoAjwGPBjIoY4z3YmNjmTZtGrGxseTLl49Fixaxdu1aSxKZUIpnFKr6s/v2OPAwgIjY+aYxGdiqVat4/PHHWbVqFblz5+b++++3BwllYsmeUYjILSLSUkRC3OHyIvIB1imgMRlSdHQ0ffr0oUaNGuzZs4fp06fTqlUrr8MyHksyUYjI68BHQHvgGxEZjPNMirWAXRprTAZ0//33M27cOHr27MmWLVto06YNThOlycySq3pqAVRW1dMiUgjY6w5vTZvQjDFpYefOnVxzzTXkz5+fYcOGkSVLFm655RavwzJBJLmqpzOqehpAVf8GtliSMCbjOHfuHK+99hrly5fn1VdfBeDWW2+1JGEukdwZRUkRietKXIBQn2FU1SoujUmnli1bRvfu3dm8eTOtW7emT58+XodkglhyieL+BMPjAhmIMSZtjB49mv79+xMaGsr8+fNp2rSp1yGZIJdcp4DfpWUgxpjAuXDhAidPniR//vzcc889HDx4kOeff548efJ4HZpJB1J8cFGwsQcXGXN5Nm7cSPfu3eOfNGcyp3/z4KKAdhgvIk1EZKuI7BCRZ5KY50ER2SQiG0Xk40DGY0xmcurUKZ599lmqVKnC5s2badasGenth6EJDv709QSAiORU1bOXMX9WYDzQEIgCfhGRuaq6yWeeMsCzQB1VPSIi1oeUMVfA6tWradWqFX/88QedO3dmxIgRhISEeB2WSadSPKMQkRoish7Y7g5XFpH/+LHuGsAOVd2pqudwuiZvkWCex4DxqnoEQFUPXFb0xpiLxJ0x3Hjjjdx4440sXbqUKVOmWJIw/4o/VU9jgWbAYQBVXYt/3YwXBXb7DEe543zdDNwsIj+KyAoRaeLHeo0xCZw/f54xY8bQoEEDYmNjKVy4MEuXLqVevXpeh2YyAH8SRRZV/TPBuFg/lkvsvv+EFaTZgDJAfaAdMFlErr5kRSLdRCRSRCJjYmL82LQxmcfKlSupUaMGTz75JLly5eLYsWNeh2QyGH8SxW4RqQGoiGQVkX7ANj+WiwKK+wwXw+kGJOE8c1Q1RlV/B7biJI6LqOokVY1Q1Yjs2bP7sWljMr4TJ07wxBNPULNmTf766y9mzpzJ/PnzKViwoNehmQzGn0TRA+gP3Aj8BdR0x6XkF6CMiJQQkRxAW2Bugnlm41ZjuT3U3gzs9C90YzK37Nmzs2TJEnr37h1/h7V14GcCwZ+rns6ratvLXbGqnheRXsC3QFZgiqpuFJFXgEhVnetOayQim3Cqswaq6uHL3ZYxmcWOHTt45ZVXGD9+PPnz52fVqlXkypXL67BMBpfiDXci8htOldAM4AtVPZ4WgSXFbrgzmdHZs2cZMWIEw4YNI0eOHMyfP5/bbrvN67BMOhLQG+5UtRTwKlAdWC8is0Xkss8wjDGps3jxYipXrsyLL75Iy5Yt2bJliyUJk6b8ujNbVX9S1T5ANeAYzgONjDEBpqoMGzaMmJgYvvnmG6ZPn84NN9zgdVgmk0mxjUJE8uHcKNcWCAfmALUDHJcxmdaFCxd49913adKkCcWLF+fDDz/k6quvJnfu3F6HZjIpf84oNuBc6TRCVUur6gBV/TnAcRmTKa1bt466devSrVs3Jk+eDECRIkUsSRhP+XPVU0lVvRDwSIzJxE6cOMHLL7/M6NGjKViwIFOnTqVjx45eh2UMkEyiEJG3VHUA8LmIXHJplD3hzpgrZ8iQIbz11lt07dqV4cOHU7hwYa9DMiZekpfHikgNVV0pIg0Sm+7Vg43s8liTUezevZuTJ08SFhbGoUOH2LJlC3Xr1vU6LJNBBeTyWFVd6b4NV9XvfF84jdrGmFQ4f/48o0aNIjw8nMcffxyAkJAQSxImaPnTmP1oIuO6XOlAjMkMVqxYQUREBAMGDKB+/fq8//77XodkTIqSa6Nog3NJbAkR+cJnUn7gaKADMyajmT9/Pvfeey833HADX3zxBS1btrS+mUy6kNxVTytxnkFRDOdJdXGOA6sDGZQxGYWqsnfvXooWLcpdd93FK6+8Qt++fcmfP7/XoRnjtxT7ego21pht0ott27bRs2dPtm3bxqZNm8iXL5/XIZlMLCCN2SKy1P17RET+9nkdEZG/UxusMRndmTNnGDJkCBUrViQyMpJnn33Wbpgz6VpyVU9xjzu1h+0a46f9+/dTr149tm/fTrt27Rg1ahTXX3+912EZ868kd3ls3N3YxYGsqhoL1AIeB/KmQWzGpBtxj+i97rrrqFevHgsWLODjjz+2JGEyBH8uj52N8xjUUsAHOPdQfBzQqIxJJy5cuMDEiRMpVaoUUVFRiAiTJ0+mYcOGXodmzBXjT6K4oKoxQCtgjKr2BooGNixjgt/atWupXbs2PXr0oEyZMvFnFcZkNP4kivMi8gDwMPClOy574EIyJripKk899RTVq1dn586dfPjhhyxatIgSJUp4HZoxAeHvndl34HQzvlNESgCfBDYsY4KXiHDkyBG6dOnC1q1b6dChg904ZzI0v+6jEJFsQGl3cIeqng9oVMmw+yiMF/7880/69u3Liy++SLVq1bhw4QJZsvj1gEhjgkJAn5ktIrcBO4B3gSnANhGpk5qNGZPexMTEMGLECMqVK8fChQvZunUrgCUJk6n48+Ci0UBTVd0EICLhwIdAqjKTMenFTz/9xOOPP86GDRto0aIFY8eO5cYbb/Q6LGPSnD+JIkdckgBQ1c0ikiOAMRkTFBYtWkR0dDSzZ8+mRYsWXodjjGdSbKMQkanAWZyzCID2QB5V7RTY0BJnbRQmUFSVDz/8kGuuuYa7776bs2fPEhMTY300mQwhoG0UQHfgN+BpYBCwE+fubGMyjC1btnDnnXfSqVMn3nvvPQBy5sxpScIYUqh6EpGKQClglqqOSJuQjEk7p0+f5rXXXuONN94gb968vPPOO3Tt2tXrsIwJKsn1HvscTvcd7YGFIpLYk+6MSdfmzZvHq6++Sps2bdiyZQvdunWzK5qMSSC5M4r2QCVVPSki1wBf4Vwea0y6tn//ftasWUOTJk144IEHCA0NpUaNGl6HZUzQSu6n01lVPQmgqgdTmNeYoBcbG8uECRMoW7YsDz/8MKdPn0ZELEkYk4LkzihK+jwrW4BSvs/OVtVWAY3MmCvo119/pXv37vzyyy/cddddTJgwwR4mZIyfkksU9ycYHhfIQIwJlN9//50aNWoQEhLCxx9/TNu2ba1vJmMuQ5KJQlW/S8tAjLmSVJX169dTqVIlSpQowXvvvce9997L1Vdf7XVoxqQ71u5gMpzff/+dZs2aUbVqVdatWwfAww8/bEnCmFQKaKIQkSYislVEdojIM8nM11pEVESs/yiTaufOnWP48OGUL1+epUuXMnLkSMqVK+d1WMake/709QSAiORU1bOXMX9WYDzQEIgCfhGRub79Rrnz5Qf6AD/7u25jEoqNjaV27dqsWlT1liAAABwWSURBVLWKVq1aMWbMGIoXL+51WMZkCP50M15DRNYD293hyiLyHz/WXQPn2RU7VfUcMB1IrGe1ocAI4Iz/YRvjOHbsGABZs2bl0UcfZd68eXz++eeWJIy5gvypehoLNAMOA6jqWpwn3qWkKLDbZziKBM/aFpGqQHFV/ZJkiEg3EYkUkUh7LrEBp7F66tSplCxZkjlz5gDQs2dPmjVr5nFkxmQ8/iSKLKr6Z4JxsX4sl9j1h/Fd1YpIFpxnXQxIaUWqOklVI1Q1Int2e1x3Zrdp0ybq169P586dCQsLo1SpUl6HZEyG5k+i2C0iNQAVkawi0g/Y5sdyUYDv+X8xYK/PcH6gArBERP4AagJzrUHbJGfEiBFUrlyZDRs2MHnyZJYtW0aFChW8DsuYDM2fRNED6A/cCPyFc0Dv4cdyvwBlRKSE+6CjtsDcuImqGq2qIaoaqqqhwAqguapGXuZnMJlA3HNTrr/+etq3b8+WLVvo0qWLdeBnTBpI8cFF/2rlIk2BMUBWYIqqDhORV4BIVZ2bYN4lwFMpJQp7cFHmsnfvXvr27cttt91Gnz59vA7HmHTr3zy4KMXLY0Xk//BpW4ijqt1SWlZVv8LpddZ33ItJzFs/pfWZzCOuA7/BgwcTExND7dq1vQ7JmEzLn/soFvm8zwXcx8VXMxlzRa1Zs4auXbuyatUqGjVqxIQJE6zB2hgPpZgoVHWG77CIfAgsDFhEJtOLjo5m7969zJgxgwceeMA68DPGY37fme2jBHDTlQ7EZF6qysyZM9m+fTuDBw/m9ttvZ+fOneTKlcvr0Iwx+Hdn9hER+dt9HcU5m3gu8KGZzOC3336jadOmtGnThjlz5hB3Q6UlCWOCR7KJQpxz/srANe6roKqWVNVP0yI4k3GdPXuWYcOGUaFCBX788UfefvttfvrpJ+yGSmOCT7JVT6qqIjJLVaunVUAmc9i9ezdDhw7l3nvvZcyYMRQtWjTlhYwxnvDnbqWVIlIt4JGYDO/gwYOMG+c8KLF06dJs2rSJmTNnWpIwJsglmShEJO5soy5OstgqIr+KyGoR+TVtwjMZwYULF3j33XcJCwujf//+bN26FYCSJUt6HJkxxh/JVT2tBKoBLdMoFpMBbdiwgR49evDDDz9w2223MXHiRMqWLet1WMaYy5BcohAAVf0tjWIxGcy5c+do1KgR586dY8qUKTzyyCN2T4Qx6VByieIaEemf1ERVHRWAeEwG8P3333P77beTI0cOPv30U8LCwggJCfE6LGNMKiXXmJ0VyIfTHXhiL2MuEhUVxf3330+DBg344IMPAKhbt64lCWPSueTOKPap6itpFolJt86fP8+4ceN44YUXiI2N5fXXX6d9+/Zeh2WMuUJSbKMwJiUPP/ww06dP5+6772b8+PGUKFHC65CMMVdQks+jEJFCqvp3GseTInseRXA4evQo2bJlI1++fPzwww/s37+f+++/3xqrjQlS/+Z5FEm2UQRjkjDeU1WmT59OeHg4L7zwAuC0Q7Ru3dqShDEZlD1H0vhtx44dNG7cmHbt2lGsWDE6dOjgdUjGmDRgicL45eOPP6ZChQr8/PPPjBs3jhUrVlC9unUBZkxmkJrnUZhMJCYmhuzZsxMREUHr1q0ZMWIEN9xwg9dhGWPSUJKN2cHKGrPTxoEDBxgwYAAnT57kiy++8DocY8y/FJDGbJM5XbhwgUmTJlG2bFlmzJhB+fLliY2N9TosY4yHrOrJxNu5cycdOnRg+fLl1K9fn//+97+EhYV5HZYxxmOWKEy8AgUKcPToUd5//30efvhhu9zVGANY1VOmN3fuXFq1akVsbCyFCxdmw4YNdOzY0ZKEMSaeJYpMateuXbRs2ZIWLVqwbds29u3bB0CWLLZLGGMuZkeFTOb8+fOMHDmS8PBwFixYwBtvvMHq1aspVqyY16EZY4KUtVFkMrGxsUyePJk777yT//znP4SGhnodkjEmyNkZRSZw5MgRBg0axPHjx8mZMyc//vgjc+fOtSRhjPGLJYoMTFX56KOPCAsL46233mLx4sUAFC5c2BqrjTF+s0SRQW3bto2GDRvSoUMHQkNDiYyMpHnz5l6HZYxJh6yNIoPq168fkZGRTJgwgW7dupE1a1avQzLGpFOWKDKQhQsXEhYWRvHixfnvf/9Lzpw5uf76670OyxiTzgW06klEmojIVhHZISLPJDK9v4hsEpF1IvKdiNwUyHgyqv379/PQQw/RqFEj3njjDQBuuukmSxLGmCsiYIlCRLIC44G7gXJAOxEpl2C21UCEqlYCPgNGBCqejOjChQtMnDiRsLAwPv/8c1566SVGjhzpdVjGmAwmkGcUNYAdqrpTVc8B04EWvjOo6mJVPeUOrgDsrq/L8Prrr9OjRw+qV6/OunXrGDJkCLly5fI6LGNMBhPINoqiwG6f4Sjg1mTm7wJ8ndgEEekGdAPIV6TUlYovXTp+/DiHDh2iRIkSdO/enRIlStCuXTu73NUYEzCBPKNI7MiV6FOSRKQDEAG8mdh0VZ2kqhGqGpE9e/YrGGL6oarMmjWLcuXK0aZNG1SVwoUL89BDD1mSMMYEVCATRRRQ3Ge4GLA34UwichcwGGiuqmcDGE+69eeff9K8eXNatWpFoUKFGDt2rCUHY0yaCWTV0y9AGREpAewB2gIP+c4gIlWBd4AmqnoggLGkW8uXL+euu+4CYOTIkfTt25ds2eyqZmNM2gnYGYWqngd6Ad8Cm4FPVXWjiLwiInG3CL8J5ANmisgaEZkbqHjSm2PHjgFQrVo1Hn30UTZv3syAAQMsSRhj0pyoJtpsELQK3RSuf/+52eswAubw4cM888wzLFiwgI0bN5IvXz6vQzLGZAAiskpVI1KzrPX1FCRUlQ8++ICwsDDee+892rRpY+0QxpigYPUYQSA6OpqWLVuyZMkSatWqxcSJE6lUqZLXYRljDGCJwlOqiohw1VVXERISwqRJk+jSpYs9jtQYE1TsiOSRb7/9lmrVqhEVFYWIMHPmTB577DFLEsaYoGNHpTS2b98+2rZtS5MmTTh16hQHDthVwcaY4GaJIg2NHz+esLAwZs+ezcsvv8y6deuoVq2a12EZY0yyrI0iDa1atYpbb72V8ePHU6ZMGa/DMcYYv9gZRQAdO3aMfv36sWrVKgAmTJjAt99+a0nCGJOuWKIIAFXls88+Izw8nLFjx7J06VIAcuXKZfdGGGPSHUsUV9jvv/9Os2bNeOCBB7j22mtZvnw5/fv39zosY4xJNUsUV9hHH33EsmXLGD16NL/88gu33prcIziMMSb4WV9PV8D//vc/zp49y1133cXZs2c5ePAgxYrZw/qMMcHD+nryyKFDh3j00UepV68er7zyCgA5c+a0JGGMyVDs8thUUFWmTp3KwIEDiY6OZtCgQbzwwgteh2WCTExMDFFRUZw5c8brUEwmkitXLooVK8aVfBqoJYpU+Oqrr3j00UepU6cOEydOpEKFCl6HZIJQVFQU+fPnJzQ01K52M2lCVTl8+DBRUVGUKFHiiq3Xqp78dOrUKX788UcAmjZtypw5c1i2bJklCZOkM2fOULhwYUsSJs2ICIULF77iZ7GWKPzw9ddfU6FCBe6++26OHj2KiNC8eXPrwM+kyJKESWuB2OfsSJeMPXv28MADD9C0aVNy5szJvHnzuPrqq70Oyxhj0pQliiQcOHCAcuXK8eWXX/Lqq6+ydu1abr/9dq/DMuayZM2alSpVqlChQgXuvfdejh49Gj9t48aN3Hnnndx8882UKVOGoUOH4nu5/Ndff01ERATh4eGEhYXx1FNPefERkrV69Wq6du3qdRhJOnz4MHfccQf58uWjV69eSc73999/07BhQ8qUKUPDhg05cuQI4LQ59OnTh9KlS1OpUiV+/fVXAA4ePEiTJk3S5DOAJYpL7NmzB4Brr72WoUOHsmHDBgYPHkyOHDk8jsyYy5c7d27WrFnDhg0bKFSoEOPHjwfg9OnTNG/enGeeeYZt27axdu1afvrpJyZMmADAhg0b6NWrF9OmTWPz5s1s2LCBkiVLXtHYzp8//6/X8dprr9G7d+803eblyJUrF0OHDmXkyJHJzjd8+HAaNGjA9u3badCgAcOHDwecZL19+3a2b9/OpEmT6NGjBwDXXHMNRYoUiW83DTS76skVHR3N888/zzvvvMOKFSuoVq0affr08Tosk0G8PG8jm/Yeu6LrLHfDVbx0b3m/569Vqxbr1q0D4OOPP6ZOnTo0atQIgDx58jBu3Djq16/PE088wYgRIxg8eDBhYWEAZMuWjZ49e16yzhMnTtC7d28iIyMREV566SXuv/9+8uXLx4kTJwD47LPP+PLLL5k6dSqPPPIIhQoVYvXq1VSpUoVZs2axZs2a+Crd0qVL8+OPP5IlSxa6d+/Orl27ABgzZgx16tS5aNvHjx9n3bp1VK5cGYCVK1fSr18/Tp8+Te7cuXnvvfcoW7YsU6dOZf78+Zw5c4aTJ0/y/fff8+abb/Lpp59y9uxZ7rvvPl5++WUAWrZsye7duzlz5gx9+/alW7dufpdvYvLmzUvdunXZsWNHsvPNmTOHJUuWANCpUyfq16/PG2+8wZw5c+jYsSMiQs2aNTl69Cj79u2jSJEitGzZko8++uiScgmETJ8oVJWZM2fSr18/9u/fT69evShVqpTXYRlzRcXGxvLdd9/RpUsXwKl2ql69+kXzlCpVihMnTnDs2DE2bNjAgAEDUlzv0KFDKVCgAOvXrweIrzJJzrZt21i0aBFZs2blwoULzJo1i86dO/Pzzz8TGhrKddddx0MPPcSTTz5J3bp12bVrF40bN2bz5ot7ZIiMjLzoqsOwsDCWLVtGtmzZWLRoEc899xyff/45AMuXL2fdunUUKlSIBQsWsH37dlauXImq0rx5c5YtW0a9evWYMmUKhQoV4vTp09xyyy3cf//9FC5c+KLtPvnkkyxevPiSz9W2bVueeeaZFD9/Yv766y+KFCkCQJEiReIfaLZnzx6KFy8eP1+xYsXYs2cPRYoUISIigueffz5V27tcmTpRqCqtWrVi9uzZVKtWjblz5xIRkao73I1J1uX88r+STp8+TZUqVfjjjz+oXr06DRs2BP55XntiLueqmUWLFjF9+vT44YIFC6a4zAMPPEDWrFkBaNOmDa+88gqdO3dm+vTptGnTJn69mzZtil/m2LFjHD9+nPz588eP27dvH9dcc038cHR0NJ06dWL79u2ICDExMfHTGjZsSKFChQBYsGABCxYsoGrVqoBzVrR9+3bq1avH2LFjmTVrFgC7d+9m+/btlySK0aNH+1c4V0BiXSzF/X+uvfZa9u7dmyZxZMpEERMTQ/bs2RER6taty5133knPnj3jd15jMoq4Noro6GiaNWvG+PHj6dOnD+XLl2fZsmUXzbtz507y5ctH/vz5KV++PKtWrYqv1klKUgnHd1zCa/rz5s0b/75WrVrs2LGDgwcPMnv27PhfyBcuXGD58uXkzp072c/mu+4XXniBO+64g1mzZvHHH39Qv379RLepqjz77LM8/vjjF61vyZIlLFq0iOXLl5MnTx7q16+f6P0IgTijuO666+KrlPbt28e1114LOGcQu3fvjp8vKiqKG264AXDKNbnyuZIyXWP2kiVLqFSpEnPmzAFgwIAB9O7d25KEydAKFCjA2LFjGTlyJDExMbRv354ffviBRYsWAc6ZR58+fXj66acBGDhwIK+99hrbtm0DnAP3qFGjLllvo0aNGDduXPxwXNXTddddx+bNm+OrlpIiItx3333079+f8PDw+F/vCde7Zs2aS5YNDw+/qO4/OjqaokWLAjB16tQkt9m4cWOmTJkS34ayZ88eDhw4QHR0NAULFiRPnjxs2bKFFStWJLr86NGjWbNmzSWv1CYJgObNm/P+++8D8P7779OiRYv48R988AGqyooVKyhQoEB8FdW2bdvS7IbfTJMoDh48SKdOnbjjjjs4e/bsRaewxmQGVatWpXLlykyfPp3cuXMzZ84cXn31VcqWLUvFihW55ZZb4i/hrFSpEmPGjKFdu3aEh4dToUIF9u3bd8k6n3/+eY4cOUKFChWoXLly/C/t4cOH06xZM+688874A1tS2rRpw7Rp0+KrnQDGjh1LZGQklSpVoly5ckycOPGS5cLCwoiOjub48eMAPP300zz77LPUqVOH2NjYJLfXqFEjHnroIWrVqkXFihVp3bo1x48fp0mTJpw/f55KlSrxwgsvULNmzZQL1Q+hoaH079+fqVOnUqxYsfgqta5duxIZGQnAM888w8KFCylTpgwLFy6MTzpNmzalZMmSlC5dmsceeyz+qjSAxYsXc88991yRGFOSKboZ/+STT3jiiSc4ceIEAwcOZPDgweTJkydAERrj2Lx5M+Hh4V6HkaGNHj2a/PnzB/W9FIFSr1495syZk2i7UGL7nnUznoLz589ToUIF1qxZw7BhwyxJGJNB9OjRg5w5c3odRpo7ePAg/fv39+vigSshQ55RnDx5kqFDh3LjjTfSs2fP+CsHrN8dk5bsjMJ4xc4oUvDll19Svnx53njjjfiGOBGxJGE8kd5+iJn0LxD7XIZJFFFRUbRq1Yp7772XvHnzsmzZMsaMGeN1WCYTy5UrF4cPH7ZkYdJM3PMocuXKdUXXm2Huo9i5cyfffvstr7/+Ov3797e+mYznihUrRlRUFAcPHvQ6FJOJxD3h7kpK120UK1euZPny5fTt2xdwempMeBelMcaYIG6jEJEmIrJVRHaIyCV3o4hIThGZ4U7/WURC/Vnv0aNH6dmzJzVr1mTUqFGcPHkSwJKEMcYEQMAShYhkBcYDdwPlgHYiUi7BbF2AI6paGhgNvJHSes+diiYsLIx33nmHPn36sH79+otuzzfGGHNlBfKMogawQ1V3quo5YDrQIsE8LYD33fefAQ0khcuTTh7aT/Hixfnll18YM2YMV1111RUP3BhjzD8C2ZhdFNjtMxwF3JrUPKp6XkSigcLAId+ZRKQbENcx/NnIyMgNCbtIzqRCSFBWmZiVxT+sLP5hZfGPsqldMJCJIrEzg4Qt5/7Mg6pOAiYBiEhkahtkMhori39YWfzDyuIfVhb/EJHI1C4byKqnKKC4z3AxIGHn6fHziEg2oADwdwBjMsYYc5kCmSh+AcqISAkRyQG0BeYmmGcu0Ml93xr4XtPb9brGGJPBBazqyW1z6AV8C2QFpqjqRhF5BYhU1bnAu8CHIrID50yirR+rnhSomNMhK4t/WFn8w8riH1YW/0h1WaS7G+6MMcakrQzT15MxxpjAsERhjDEmWUGbKALV/Ud65EdZ9BeRTSKyTkS+E5GbvIgzLaRUFj7ztRYRFZEMe2mkP2UhIg+6+8ZGEfk4rWNMK358R24UkcUistr9njT1Is5AE5EpInJARDYkMV1EZKxbTutEpJpfK1bVoHvhNH7/BpQEcgBrgXIJ5ukJTHTftwVmeB23h2VxB5DHfd8jM5eFO19+YBmwAojwOm4P94sywGqgoDt8rddxe1gWk4Ae7vtywB9exx2gsqgHVAM2JDG9KfA1zj1sNYGf/VlvsJ5RBKT7j3QqxbJQ1cWqesodXIFzz0pG5M9+ATAUGAGcScvg0pg/ZfEYMF5VjwCo6oE0jjGt+FMWCsT191OAS+/pyhBUdRnJ34vWAvhAHSuAq0WkSErrDdZEkVj3H0WTmkdVzwNx3X9kNP6Uha8uOL8YMqIUy0JEqgLFVfXLtAzMA/7sFzcDN4vIjyKyQkSapFl0acufshgCdBCRKOAroHfahBZ0Lvd4AgTvg4uuWPcfGYDfn1NEOgARwO0Bjcg7yZaFiGTB6YX4kbQKyEP+7BfZcKqf6uOcZf5PRCqo6tEAx5bW/CmLdsBUVX1LRGrh3L9VQVUvBD68oJKq42awnlFY9x//8KcsEJG7gMFAc1U9m0axpbWUyiI/UAFYIiJ/4NTBzs2gDdr+fkfmqGqMqv4ObMVJHBmNP2XRBfgUQFWXA7lwOgzMbPw6niQUrInCuv/4R4pl4Va3vIOTJDJqPTSkUBaqGq2qIaoaqqqhOO01zVU11Z2hBTF/viOzcS50QERCcKqidqZplGnDn7LYBTQAEJFwnESRGZ9ROxfo6F79VBOIVtV9KS0UlFVPGrjuP9IdP8viTSAfMNNtz9+lqs09CzpA/CyLTMHPsvgWaCQim4BYYKCqHvYu6sDwsywGAP8nIk/iVLU8khF/WIrIJzhVjSFue8xLQHYAVZ2I0z7TFNgBnAI6+7XeDFhWxhhjrqBgrXoyxhgTJCxRGGOMSZYlCmOMMcmyRGGMMSZZliiMMcYkyxKFCToiEisia3xeocnMG5pUT5mXuc0lbu+ja90uL8qmYh3dRaSj+/4REbnBZ9pkESl3heP8RUSq+LFMPxHJ82+3bTIvSxQmGJ1W1So+rz/SaLvtVbUyTmeTb17uwqo6UVU/cAcfAW7wmdZVVTddkSj/iXMC/sXZD7BEYVLNEoVJF9wzh/+JyK/uq3Yi85QXkZXuWcg6ESnjju/gM/4dEcmawuaWAaXdZRu4zzBY7/b1n9MdP1z+eQbISHfcEBF5SkRa4/S59ZG7zdzumUCEiPQQkRE+MT8iIv9JZZzL8enQTUT+KyKR4jx74mV3XB+chLVYRBa74xqJyHK3HGeKSL4UtmMyOUsUJhjl9ql2muWOOwA0VNVqQBtgbCLLdQfeVtUqOAfqKLe7hjZAHXd8LNA+he3fC6wXkVzAVKCNqlbE6cmgh4gUAu4DyqtqJeBV34VV9TMgEueXfxVVPe0z+TOglc9wG2BGKuNsgtNNR5zBqhoBVAJuF5FKqjoWpy+fO1T1Drcrj+eBu9yyjAT6p7Adk8kFZRceJtM77R4sfWUHxrl18rE4/RYltBwYLCLFgC9UdbuINACqA7+43Zvkxkk6iflIRE4Df+B0Q10W+F1Vt7nT3weeAMbhPOtisojMB/zu0lxVD4rITrefne3uNn5013s5cebF6a7C9wllD4pIN5zvdRGcB/SsS7BsTXf8j+52cuCUmzFJskRh0osngb+Ayjhnwpc8lEhVPxaRn4F7gG9FpCtOt8rvq+qzfmyjvW8HgiKS6PNN3L6FauB0MtcW6AXceRmfZQbwILAFmKWqKs5R2+84cZ7iNhwYD7QSkRLAU8AtqnpERKbidHyXkAALVbXdZcRrMjmrejLpRQFgn/v8gIdxfk1fRERKAjvd6pa5OFUw3wGtReRad55C4v8zxbcAoSJS2h1+GFjq1ukXUNWvcBqKE7vy6DhOt+eJ+QJoifOMhBnuuMuKU1VjcKqQarrVVlcBJ4FoEbkOuDuJWFYAdeI+k4jkEZHEzs6MiWeJwqQXE4BOIrICp9rpZCLztAE2iMgaIAznkY+bcA6oC0RkHbAQp1omRap6Bqd3zZkish64AEzEOeh+6a5vKc7ZTkJTgYlxjdkJ1nsE2ATcpKor3XGXHafb9vEW8JSqrsV5PvZGYApOdVacScDXIrJYVQ/iXJH1ibudFThlZUySrPdYY4wxybIzCmOMMcmyRGGMMSZZliiMMcYkyxKFMcaYZFmiMMYYkyxLFMYYY5JlicIYY0yy/h/6cAJgUscOzwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred = cnn.predict(x_test[0:100])\n",
    "pred = pred[:,1] # Only positive class (M)\n",
    "plot_roc(pred,y_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnn.load_weights('data/best_weights.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[43  1]\n",
      " [ 0 56]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAEYCAYAAAAaryJBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAf00lEQVR4nO3dd5xkRb3+8c+zuyAgOYqggooYUBAWLoIoQblgAgNBkQuIclExgQERSdesPzOGRZQ1wwUxgArIBRREwi5ZVARFCZJBQEB2eX5/nBrpHWd6ume7+5zped68+jV9Up3qpV/znW9VnSrZJiIiYtBm1F2BiIiYnhKAIiKiFglAERFRiwSgiIioRQJQRETUYlbdFYiIiP6ZufyT7AUPdHWNH7jtNNs79KlK/5IAFBExxLzgAR6z/q5dXfPgpUev2qfqLCIBKCJiqAnUzN6WBKCIiGEmQKq7FmNKAIqIGHbJgCIiohbJgCIiYvDSBxQREXVJBhQREQMnkgFFREQdlAwoIiJqkgwoIiJqkQwoIiIGL6PgIiKiDpkJISIiapMMKCIiBi9NcBERUZcZaYKLiIhBy4OoERFRmwxCiIiIwUsfUERE1CUZUERE1CIZUEREDJwyGWlERNQlGVBERNQiGVBERAxef0bBSfozcC+wEFhge7aklYHjgXWAPwO72r5rvDKamZdFRETvjPQDdfrq3Da2N7I9u2wfDJxpez3gzLI9rgSgiIhhNjITQjevydsJmFvezwV2bndyAlBExFDTZALQqpIubnntN0bBBk6XNK/l+Bq2bwYoP1dvV7P0AUVEDLvuByHc3tKsNp4tbd8kaXXgDEm/6/YmCUAREcOuD4MQbN9Uft4q6WRgM+AWSWvavlnSmsCt7cpIE1xExLDr8SAESY+VtNzIe2B74Ergx8Be5bS9gB+1KycZUETEMFNfhmGvAZysKljNAr5r++eSLgJOkLQv8Bdgl3aFJABFRAy7Hj+Iavs6YMMx9t8BbNdpOQlAERFDTpkJISIiBk0kAEVERB1UXg2UUXAREVGLZEAREUNNaYKLiIh6JABFREQtEoAiIqIWCUARETF4GQUXUS9JS0v6iaR7JP3vYpSzh6TTe1m3ukjaStLv665H9JfKIIRuXoOSABSNIul1Zf2R+yTdLOlnkp7fg6JfQzV/1Sq2285P1Y7t79jevgf16StJlvTUdufY/pXt9QdVp6hPAlDEBCQdCHwW+AhVsHgi8CWqVRYX15OAP9he0IOypjxJaX6fRhKAItqQtAJwFPBW2z+wfb/th23/xPZ7yjmPkfRZSTeV12clPaYc21rSDZIOknRryZ72KceOBA4DdiuZ1b6SjpD07Zb7r1Oyhllle29J10m6V9KfJO3Rsv/cluu2kHRRadq7SNIWLcfOlvQ/ks4r5ZwuadVxPv9I/d/bUv+dJb1E0h8k3SnpkJbzN5N0vqS7y7lflLRkOfbLctpl5fPu1lL++yT9DfjGyL5yzVPKPTYu24+XdLukrRfrf2w0QgJQRHvPA5YCTm5zzgeAzYGNqGbi3Qw4tOX444AVgLWAfYGjJa1k+3CqrOp428vaPrZdRVStb/J5YEfbywFbAJeOcd7KwKnl3FWATwOnSlql5bTXAftQLU28JPDuNrd+HNW/wVpUAfMY4PXAJsBWwGGSnlzOXQi8C1iV6t9uO+AtALZfUM7ZsHze41vKX5kqG1xkiWXb1wLvA74jaRngG8Bxts9uU9+YCjSJ14AkAEVTrEK1DHC7JrI9gKNs32r7NuBIYM+W4w+X4w/b/ilwHzDZPo5HgA0kLW37ZttXjXHOS4FrbH/L9gLb3wN+B7y85Zxv2P6D7QeAE6iC53geBj5s+2Hg+1TB5XO27y33vwp4DoDtebZ/U+77Z+CrwAs7+EyH236o1GcRto8BrgEuANakCvgxBJIBRbR3B7DqBH0Tjweub9m+vuz7VxmjAtg/gGW7rYjt+4HdgP2BmyWdKunpHdRnpE5rtWz/rYv63GF7YXk/EiBuaTn+wMj1kp4m6RRJf5P0d6oMb8zmvRa32X5wgnOOATYAvmD7oQnOjSkgo+AiJnY+8CCwc5tzbqJqPhrxxLJvMu4HlmnZflzrQdun2X4xVSbwO6pfzBPVZ6RON06yTt34MlW91rO9PHAIEzeeuN1BSctSDQI5FjiiNDHGEEgAimjD9j1U/R5Hl873ZSQtIWlHSZ8op30POFTSaqUz/zDg2+OVOYFLgRdIeqKqARDvHzkgaQ1Jryh9QQ9RNeUtHKOMnwJPUzV0fJak3YBnAqdMsk7dWA74O3Bfyc7ePOr4LcCT/+2q9j4HzLP9Rqq+ra8sdi2jGdIHFNGe7U8DB1INLLgN+CtwAPDDcsqHgIuBy4ErgPll32TudQZwfClrHosGjRnAQVQZzp1UfStvGaOMO4CXlXPvAN4LvMz27ZOpU5feTTXA4V6q7Oz4UcePAOaWUXK7TlSYpJ2AHaiaHaH6/7Cxyui/mMLU3AxIdtusPCIiprAlVnuKV9n5411dc8vXdplne3afqvQveRgtImLIDTKr6ca0aoJT9WBg36N6RERTNHkU3JTJgCTNyjQqERGT0MwEaLABSNI6wM+Ac6meLr+Rap6v9alG3CwDXAu8wfZdks4Gfg1sCfxY0rOpnoV4OtXw132AvaieBL/A9t7lPl8GNgWWBk4sT8JHREw/am4TXB0Z0HrAa22/SdIJwKupRg+9zfY5ko4CDgfeWc5f0fYLASQdB6wEbAu8AvgJVXB6I3CRpI1sXwp8wPadkmYCZ0p6ju3Lx6uQpP0oU5No1lKbzFpprfFOjZjQBk9Yse4qxBC4ZP68222v1ouyEoAe9acSJKAa/voUqiBzTtk3F2hdr2X08NKf2LakK4BbbF8BIOkqYB2q5zt2LUFlFtWDhM+kGm47JttzgDkAS67+VD9ut08vxseL6e6Xn3pZ3VWIIbDcUjNHz7IxaQlAj2qd3mMhMNGfi/ePc/0jo8p6BJglaV2qZyQ2Lc14x1FN8BgRMT01M/40YhTcPcBdkrYq23sC57Q5fyLLUwWteyStAey4mPWLiJjSMgquvb2Ar6iaBv46qsEFk2L7MkmXUM0cfB1wXm+qGBEx9Qw6qHRjoAGoTBu/Qcv2p1oObz7G+VuP2t67TVl7j/W+XXkREdNBAlBERNQiASgiIurRzPiTABQRMeySAUVExOBlJoSIiKiDgIbGnwSgiIjhlmHYERFRk4bGnwSgiIhh19QMqAlT8URERL+oyoC6eXVctDRT0iWSTinb60q6QNI1ko6XtGS76xOAIiKGmIAZM9TVqwvvAK5u2f448Bnb6wF3Afu2uzgBKCJiyPUjA5K0NvBS4GtlW1RrtZ1YTpkL7NyujPQBRUQMuUn0Aa0q6eKW7Tll3bRWn6VaTHS5sr0KcLftBWX7BqDt6p4JQBERw6zLfp3idtuzxy1Sehlwq+15krZ+9E7/xu1ukgAUETHEqgdRez4KbkvgFZJeQrXg5/JUGdGKkmaVLGht4KZ2haQPKCJiqHW3GF0nwcr2+22vbXsdYHfg/2zvAZwFvKacthfwo3blJABFRAy5fg3DHsP7gAMl/ZGqT+jYdienCS4iYsj180FU22cDZ5f31wGbdXptAlBExDBb/KymbxKAIiKGWJ8GIfREAlBExJBraPxJAIqIGHbJgCIiohYNjT8JQBERQy1LckdERB2yJHdERNQkS3JHRERNGhp/EoAiIoZdMqCIiBi8zIQQERF1yEwIERFRmwSgiIioRUPjTwJQRMSwa2oGlAXpIiKiFsmAIiKGWUbBRUREHZSZECIioi4NjT8JQBERw25GQyNQAlBExJBraPxJAIqIGGbKekAREVGXGc2MPwlAERHDLhlQRETUoqHxJwEoImKYiepZoCYaNwBJWr7dhbb/3vvqREREr03FPqCrAMMioXNk28AT+1iviIjoBU3BmRBsP2GQFYmIiP5oaPzpbDZsSbtLOqS8X1vSJv2tVkRE9IKoZkLo5jUoEwYgSV8EtgH2LLv+AXyln5WKiIjekbp7DUono+C2sL2xpEsAbN8pack+1ysiInpkyvUBtXhY0gyqgQdIWgV4pK+1ioiInhh0VtONTgLQ0cBJwGqSjgR2BY7sa60iIqJnpuxs2La/KWke8KKyaxfbV/a3WhER0SvNDD+dz4QwE3iYqhmuo5FzERHRDE3tA+pkFNwHgO8BjwfWBr4r6f39rlhERCy+ahh2d69B6SQDej2wie1/AEj6MDAP+Gg/KxYRET3Qh5kQJC0F/BJ4DFUcOdH24ZLWBb4PrAzMB/a0/c/xyumkOe16Fg1Us4DrJlvxiIgYrD48B/QQsK3tDYGNgB0kbQ58HPiM7fWAu4B92xXSbjLSz1D1+fwDuErSaWV7e+DcjqoYERG163UGZNvAfWVzifIysC3wurJ/LnAE8OXxymnXBDcy0u0q4NSW/b/pvroREVGHkT6gLq0q6eKW7Tm25yxSrjSTqjvmqVSP61wL3G17QTnlBmCtdjdpNxnpsV1XOSIiGmcSGdDttme3O8H2QmAjSSsCJwPPGOu0dmVMOAhB0lOADwPPBJZqufnTJro2IiLq18+BbbbvlnQ2sDmwoqRZJQtaG7ip3bWdDEI4DvgG1WfYETiBapRDREQ0nNT72bAlrVYyHyQtTTVRwdXAWcBryml7AT9qV04nAWgZ26cB2L7W9qFUs2NHRMQU0IdRcGsCZ0m6HLgIOMP2KcD7gAMl/RFYBWjbldPJc0APqWpAvFbS/sCNwOodVTEiImrXh1FwlwPPHWP/dcBmnZbTSQB6F7As8HaqvqAVgDd0eoOIiKhXQ2fi6Wgy0gvK23t5dFG6iIiYAsRgVzntRrsHUU+mzRA626/qS40iIqJ3puh6QF8cWC0a5NlPWJHzPvOKuqsRU9hKmx5QdxUiFtHU2bDbPYh65iArEhER/dHUNXQ6XQ8oIiKmIDEFM6CIiBgOg1zjpxsdByBJj7H9UD8rExERvdfUANTJiqibSboCuKZsbyjpC32vWURELLZqdgN19RqUTvqmPg+8DLgDwPZlZCqeiIgpYyovyT3D9vWjouLCPtUnIiJ6rKFjEDoKQH+VtBngsgDR24A/9LdaERHRC9WCdM2MQJ0EoDdTNcM9EbgF+EXZFxERU8CUfQ7I9q3A7gOoS0RE9EFDE6COVkQ9hjHmhLO9X19qFBERPaMOF5mrQydNcL9oeb8U8Ergr/2pTkRE9FpD409HTXDHt25L+hZwRt9qFBERPdXUB1EnMxXPusCTel2RiIjovSk9Ck7SXTzaBzQDuBM4uJ+VioiI3mlo/GkfgFQ9fbohcGPZ9YjtcRepi4iIhhnw7AbdaDs8vASbk20vLK8En4iIKUZd/jconTyfdKGkjftek4iI6LmqD2iKzQUnaZbtBcDzgTdJuha4n+rz2HaCUkTEFNDUJrh2fUAXAhsDOw+oLhER0QdTcUVUAdi+dkB1iYiIHhtpgmuidgFoNUkHjnfQ9qf7UJ+IiOglTc1h2DOBZWGAQyIiIqLnpuKDqDfbPmpgNYmIiJ5rchNcu2HYDa1yREQMg3YZ0HYDq0VERPRNQ1vgxg9Atu8cZEUiIqIfxIyGNmhNZjbsiIiYIsQUzIAiImIINHgy0gSgiIghNxWHYUdExBSXJriIiKhNMqCIiKhFQ+NPAlBExDATnS38Voem1isiInpB1XIM3bwmLFJ6gqSzJF0t6SpJ7yj7V5Z0hqRrys+V2pWTABQRMeTU5asDC4CDbD8D2Bx4q6RnAgcDZ9peDzizbI8rASgiYohVk5Gqq9dEbN9se355fy9wNbAWsBMwt5w2lwkWNE0fUETEkJvEGIRVJV3csj3H9pwxy5bWAZ4LXACsYftmqIKUpNXb3SQBKCJiyE1iFNzttmdPXK6WBU4C3mn7790u/Z0AFBEx1DobWNB1qdISVMHnO7Z/UHbfImnNkv2sCdzaroz0AUVEDLGRYdjdvCYss4poxwJX2/50y6EfA3uV93sBP2pXTjKgiIgh14cMaEtgT+AKSZeWfYcAHwNOkLQv8Bdgl3aFJABFRAy5Xocf2+e2KbbjxUwTgCIihpn6kgH1RAJQRMQQa/JUPAlAERFDLhlQRETUopnhJwEoImLoNTQBSgCKiBhmVR9QMyNQAlBExJBLBhQRETUQSgYUERF1SAYUEREDlz6giIioh5IBRURETRKAIiKiFhmEEBERAydgRjPjTwJQRMSwSwYUERG1SB9QRETUIhlQREQMXPqAIiKiJpmKJyIi6pAHUSMioi4NjT8JQBERw6zqA2pmCEoAiogYcs0MPwlAERHDr6ERKAEoImLIZRRcRETUoqFdQMzoV8GSft3BOe+UtEwP7vVnSasubjkREcNIXb4GpW8ByPYWHZz2TmDMACRpZm9rNJiyIyIap6ERqJ8Z0H3l59aSzpZ0oqTfSfqOKm8HHg+cJemskWskHSXpAuB5rZmNpNmSzi7vV5F0uqRLJH2Vln8ySa+XdKGkSyV9dSTYjC67X587IqJJqpjS3X+DMqg+oOcCzwJuAs4DtrT9eUkHAtvYvr2c91jgStuHAWj8hsvDgXNtHyXppcB+5fxnALuV8h+W9CVgD+Cbo8tuJWm/kTKA+5ZeQr9f7E883FYFbp/wrIjx5Ts0sSf1pJTMhMCFtm8AkHQpsA5w7hjnLQRO6qC8FwCvArB9qqS7yv7tgE2Ai0rwWhq4daKybc8B5nTyQQIkXWx7dt31iKkr36HBamj8GVgAeqjl/cI2933Q9sKW7QU82ky41KhzPcb1Aubafn8HZUdETA8NjUB96wPq0L3Acm2O/5kqowF4dcv+X1I1rSFpR2Clsv9M4DWSVi/HVpbUmzQ2ImJK6rYHaHDRqu4ANAf42cgghDEcCXxO0q+oMqfW/S+QNB/YHvgLgO3fAocCp0u6HDgDWLNflZ/G0lwZiyvfoQGSunsNrF72WC1ZERExDJ71nI393VPO6eqajZ60/LxB9NFlJoSIiGHX0D6gBKCIiCHX1Lng6u4DioiIPut1H5Ckr0u6VdKVLftWlnSGpGvKz5XalQEJQNEHkvK9isWmlifR1eap9JhYH2biOQ7YYdS+g4Ezba9HNSL54IkKyS+K6ClJmwL79GKS2Zi+JM1wGSEl6dVUM6nEZHQbfTqIQLZ/Cdw5avdOwNzyfi6w80TlJABFry0L7A/sKmnpuisTU5PtRwAkbQ7sCdxYb42mnVUlXdzy2m/iS1jD9s0A5efqE12QQQjRU7bPkvQeqvn6Zkr6ru0H6q5XTC2lyW1r4HTgANt3SVrK9oP11mxqmsQghNsHMQw7GVAsttHt87bPBo6g+sv1dcmEohOt3yNXzgK+DXyw7HswS6l0TwzsQdRbJK0JUH7eOsH5CUCxeCSppa1+F0kHSZpt+xzg/VRBaPf0CUU7o75Hr5S0d/ke7QP8X1leZabthQlC3RvQckA/BvYq7/cCfjTRBQlAsVhafmkcQLXA4CPAtyS9BbiAaiTM21l0Lr+IRbR8jw6i+r48Hjha0n/a/i/gUuBPZXBCJhXuVo8jkKTvAecD60u6QdK+wMeAF0u6Bnhx2W4rfUCx2CRtDGxDtRzGvsA/ga2AWWXdpzcBt9VYxWgoSSvbvrM0vz0OeK7tbUo/4h3A2QC295b0FaqlXK6rq75TVa8fRLX92nEObddNOcmAomtj9PnMB94KvBB4pe0NqdZ7ep+kfW1fbPv6GqoaDVVWRV4LmCdp+5IBPViOnUD1XdrJ9kOS9pS0lu39bSf4TEJTJyNNAIqutTSX7ChppzI66W/AysDd5bQbqZbNOKWmakazyfaNwCHA/5P0Att3AZdRZTlHlFWN9wHeR2NnM5saBtQH1LU0wUXHRnUUvxF4G9WaTttJ+jpwDrC/pJ9TLYOxi+1baqtwNNbIcz7ADVQPNP5U0suBE6jWCDu6LKmyBbDryIrKMUkNDd8JQNGRUcFnKaoAsxVVAPoU8Hrge8BrqZZMvzDNJdGOpDcAbyqv1wDfoQo2h0k6hWoV5KNs/7XGak55VVbTzAiUABQTGhV83g28CHgacLXtEyV9BPgA1S+SL9j+fn21jaZq/R4VawMn2b4SuFLSjVSZ0K62f15PLYfQgPt1upE+oJhQS/B5IfB84L3A0cBhkra1fQfwEaqmlNtrq2g0Wsv3aNOy60bgqS3HjwHmA0fk4eXeSh9QTDmjMp+tqfp8brJ9OXC5pLuAT0l6v+3TJB3a0rYf8W8krQZ8QtLPgM8Be0r6KHA8sAFwCfDpTN/UY8mAYioZFXxeR/XL4bfA6pKeL2mW7a8DXwM+mJkOokN3UWXQWwH7AC8BVqD64+YA4Jj0+fSauv5vUJIBxZhags9s4NW2X122PwzsXr3V+ba/JOk7tv9RY3WjgUb9EbMb8FvbV0iaDxwGfBR4yPZbyjkr2L6nvhoPr/QBxZRSHhTcEDgWeFDSY8uhI6n6evYFNgPIL40YbVTweQawJXCSpGeVqXSuAn5BlT2/q1z293pqO9z6sBxQzyQAxb+MMRvxZVRDrNcBNpa0pO1/Ah8G/kimRIlxtASf7YHP2X471R8z35e0Qfke3QJ8Czip9Zrog4ZGoDTBxb+0/NLYA1iPajr1bwMLqJpMjpR0oe2HgA/VVtForFGZzz5Us6GPLKfw8fJHzjckXUI13c4Otv9SW4WniaY+B5QMKBYh6a1UHcJ3AesDp5XXXKpsaJP6ahdNViYWHQk+T6aaLfmFVDMjA2D7Y8B7gJ8AL7f9pzrqOt00dS64ZEDT3MhfrC1/uT4beLvtC8vxQ4BP2H6jpBXI0sgxvpeUZ3z+CLze9n+UZ8dOl/SXMmpyZMHCGKBm5j/JgKa1UU+mrydpCaqn07duOe0UyvfE9tFpLonx2P421bIc/wO8ruw7l2rmjI+X7DoGrcvsJ7NhR9+Naqs/APgp1WwGlwFvL/N0QZURrSNpxdHLMESM8Z34InAh1WwGywDY/jXwcuCgfI/q0sxRCAlA01RL8HkF8BzgP6lGtf2danjshyR9gWpZ7bfZvjujlKLVqD9iXippO2Cu7e2pZrSeW47tDRhYP9+jwRPNzYDSBzSNlQXBvgj8wva1ZUmFkaWz/wbMoVqX5Y666hjNUzIYjUy7VEa7HUQ1cOVKSV+xvbOkH0n6MdVgllfbfri+Wk9vTU05kwFNY2VBsHcCO0javQyv/j7V8tmPAHcm+MQYZrYEn5cCr6Jqqt2WahntfSQ92/ZOwFHANmXG66hJUzOgBKBpzvYPqJZROKQEoUeA44CP2r677cUx7ZTJRH9QZsqYBcymGmq9WclwPgfcB7xN0mZlOfabaqxy0P1scIOSJrjA9qmSHgHmSFpg+0SqheYiFmH7Nkm7U41su8j2kZKWBA6WdLjty0vf4X8Df66zrtGioW1wyYACANs/A95AtR5LxLjKxLOPpervWQE4FLgIOFzSxmUZ9g/ZvrXOesajmjkGLgEoWtg+I8toRyds/xB4I3AxsDzVzNZXUw21fgzVqLdogG77fzIKLiIaz/bPJb0D+A3wPNuHSlqlDGaJBmnqXHAJQBExabZ/WvqAzpQ0O6MmG6qZ8ScBKCIWj+0fSvpFHjBtrobGnwSgiFh8tu+ruw4xvqZOfpQAFBEx1Ab7bE83EoAiIobYyFxwTZRh2BERUYtkQBERQy4ZUESfSFoo6VJJV0r635F1aCZZ1taSTinvXyHp4DbnrijpLZO4xxGS3t3p/lHnHCfpNV3cax1JmQh0mmvqXHAJQDEMHrC9ke0NgH8C+7ceLBNndv1dt/1j2x9rc8qKQNcBKGKgGjwTQgJQDJtfAU8tf/lfLelLVPPbPUHS9pLOlzS/ZErLAkjaQdLvJJ1LtbQAZf/ekr5Y3q8h6WRJl5XXFsDHgKeU7OuT5bz3SLpI0uWSjmwp6wOSfi/pF1Tr47Ql6U2lnMsknTQqq3uRpF9J+oOkl5XzZ0r6ZMu9/3tx/yFjOHQ7D1zmgouYhLI8wI7AFWXX+sA3bT8XuJ9q0swX2d6Yag6zAyUtBRxDtWT0VsDjxin+88A5tjcENgauAg4Gri3Z13skbQ+sB2wGbARsIukFkjYBdgeeSxXgNu3g4/zA9qblflcD+7YcW4dqCYSXAl8pn2Ff4B7bm5by3yRp3Q7uE9NBQyNQBiHEMFha0qXl/a+AY4HHA9fb/k3ZvznwTOC8akFPlgTOB54O/Mn2NQCSvg3sN8Y9tgX+C8D2QuAeSSuNOmf78rqkbC9LFZCWA04us0hTVgmdyAaSPkTVzLcscFrLsRPKuk3XSLqufIbtgee09A+tUO79hw7uFUMuzwFF9M8Dtjdq3VGCzP2tu4AzbL921Hkb0buZm0W1kN9XR93jnZO4x3HAzrYvk7Q3sHXLsdFludz7bbZbAxWS1unyvjGEMgouol6/AbaU9FQASctIehrwO2BdSU8p5712nOvPBN5crp0paXmqRfuWaznnNOANLX1La0laHfgl8EpJS0tajqq5byLLATdLWgLYY9SxXSTNKHV+MvD7cu83l/OR9DRJj+3gPjENNLQFLgEopgfbtwF7A9+TdDlVQHq67QepmtxOLYMQrh+niHcA20i6ApgHPKvM/HxeGf79SdunA98Fzi/nnQgsZ3s+cDxwKXASVTPhRD4IXACcQRUkW/0eOAf4GbB/+QxfA34LzC/Drr9KWjhiRB8iUBm883tJf2z3uELbMjKBbUTE8Np4k9k+7zcXd3XNMktqnu3Z4x2XNJOqf/HFwA1UK+K+1vZvu7lPMqCIiCE2Mhdcj58D2gz4o+3rbP8T+D6wU7d1S4oeETHE5s+fd9rSS2jVLi9bSlJr2jTH9pyW7bWAv7Zs3wD8R7d1SwCKiBhitnfoQ7Fj5Uld9+ekCS4iIrp1A/CElu21gZu6LSQBKCIiunURsJ6kdSUtSTXTRycPWC8iTXAREdEV2wskHUD1/NlM4Ou2r+q2nAzDjoiIWqQJLiIiapEAFBERtUgAioiIWiQARURELRKAIiKiFglAERFRiwSgiIioxf8HHPQZudptgp4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.98      0.99        44\n",
      "           1       0.98      1.00      0.99        56\n",
      "\n",
      "    accuracy                           0.99       100\n",
      "   macro avg       0.99      0.99      0.99       100\n",
      "weighted avg       0.99      0.99      0.99       100\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Plot non-normalized confusion matrix\n",
    "y_true = np.argmax(y_test[0:100],axis=1)\n",
    "pred = cnn.predict(x_test[0:100])\n",
    "pred = np.argmax(pred,axis=1)\n",
    "\n",
    "\n",
    "cm = confusion_matrix(y_true, pred)\n",
    "print(cm)\n",
    "plt.figure()\n",
    "plot_confusion_matrix(cm, ['normal','intruder'])\n",
    "plt.show()\n",
    "print(classification_report(y_true, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score (RMSE): 0.1\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD4CAYAAADhNOGaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZhU9b3n8fe3qkFiBImCN2yGzgwmEoMbLolbYlzA8UJMzIhLojGKElAT88zATSbLNT43y01cx+iFGLcYUUhUboaMTgxZFHWaDoogQbmK2kHHDqhREek65zt/1KmmuqnuPkAdqz2/z+t5eKg6dbr6d6q6+tPf3/cs5u6IiEi4Co0egIiINJaCQEQkcAoCEZHAKQhERAKnIBARCVxTowewvYYNG+Zjx45t9DBERN5VWltb/+buw2s99q4LgrFjx7Js2bJGD0NE5F3FzJ7r6TFNDYmIBE5BICISOAWBiEjgFAQiIoFTEIiIBC6zIDCzn5nZy2a2sofHzcyuMbO1ZrbCzA7KaiwiItKzLCuCm4FJvTw+GRiX/JsOXJ/hWEREpAeZHUfg7n80s7G9rDIVuNXL58F+xMyGmtkId38xqzEF6an74R/Gw+6jGz2SfisqlWi581/grVcbPRSRXu1x0FT2OeiYuj9vIw8oGwW8UHW/LVm2TRCY2XTKVQN77733OzK43LjrC3D4DDju240eSb/13F9aOfzpHwMQuzV4NCI9axkyAnIWBLU+cTWvkuPuc4G5ABMnTtSVdLZHaTNEWxo9in6ttGUzAI8f/W/sf+y0Bo9GpGeHZfS8jdxrqA0YU3V/NLC+QWPJpzgGHOJSo0fSr8VRBwBm77ozrojURSODYBHwhWTvocOB19QfqLNKACgIehVH5dfHigMaPBKRxsjsTyAzuwP4BDDMzNqAbwMDANz9BmAxcBKwFtgEfDGrsQRLQZBKpSIoFFURSJiy3Gvo9D4ed2BmVt9fAI/K/8dRY8fRz3kUA1BoUhBImHRkcZ51VgQKgt54pUegikACpSDIs0oAaGqoV3Hy+hTVI5BAKQjyTD2CVDxpFqtHIKFSEOSZgiCVytRQoUkVgYRJQZBn6hGkUpkaKmhqSAKlIMgz9QjSiSo9Ak0NSZgUBHmmqaFUKgeUaWpIQqUgyDMFQTqVvYZ0HIEESkGQZ+oRpFLZa6ioikACpSDIM1UE6eg4AgmcgiDP1CxOxWP1CCRsCoI8U0WQTvL6NKlHIIFSEOSZegSpqEcgoVMQ5JkqgnQ6KwIFgYRJQZBn6hGkk7xOOqBMQqUgyDNVBOnEJUpewAr6OEiY9JOfZ+oRpBNHRBQbPQqRhlEQ5JkqglQs7iDSR0ECpp/+PFOPIB2PKJkqAgmXgiDPVBGkYnFJU0MSNAVBnqlHkI6rRyBhUxDkmSqCVCwuEeujIAHTT3+eKQhSMe01JIFTEOSZmsWpmJeI1CyWgCkI8qwSAB6Be2PH0o+ZR8QKAgmYgiDPqisBNYx7ZHFErKkhCZiCIM+6BIGmh3qiqSEJnYIgz6qrAAVBjwquikDCpiDIM1UEqahHIKFTEOSZegSpFLykIJCgKQjyTBVBKqoIJHSZBoGZTTKzNWa21szm1Hh8bzNbYmbLzWyFmZ2U5XiCox5BKgWPcAWBBCyzIDCzInAdMBkYD5xuZuO7rfY/gLvc/UBgGvCTrMYTJFUEqRQ8IjZdnUzClWVFcCiw1t2fcfctwHxgard1HBiS3N4dWJ/heMKjIEhFFYGELssgGAW8UHW/LVlW7TvAWWbWBiwGLqr1RGY23cyWmdmy9vb2LMaaT2oWp1JAPQIJW5ZBYDWWdT/PwenAze4+GjgJuM3MthmTu89194nuPnH48OEZDDWnVBGkUvQI19SQBCzLIGgDxlTdH822Uz9fAu4CcPeHgUHAsAzHFBY1i1MpEOEFVQQSriyDoAUYZ2bNZjaQcjN4Ubd1ngc+BWBm+1IOAs391IsqglTUI5DQZRYE7l4CZgH3Aasp7x20yswuM7MpyWpfA843s8eBO4Bz3HWazLpRjyAVTQ1J6DL96Xf3xZSbwNXLvlV1+0ngiCzHEDRVBKkUNTUkgdORxXmmHkEqBWK8oIpAwqUgyDNVBKkUiUA9AgmYgiDP1CNIpTw1pIpAwqUgyLO4BE2Dtt6WmppcFYGETUGQZ3EETbsktxUEPSkS44UBjR6GSMMoCPJMFUEqRSLQXkMSMAVBnsUlVQR98DimyWJQj0ACpiDIsy4VgZrFtURREpCqCCRgCoI8iyMoqiLoTanUUb6hikACpiDIs7gETQO33pZtREkQmCoCCZiCIM/ULO5TqVSZGlJFIOFSEORZ9e6jHjd2LP2URwoCEQVBnsWRKoI+VHoEVlQQSLgUBHmm3Uf7FCcVgakikIApCPJMPYI+RaUt5RuqCCRgCoI8U0XQJ1UEIgqCfOtyHIEOKKslUo9AREGQa3EJigPACqoIerC1ItBJ5yRcCoI8i0vl3SILTQqCHkTJcQSqCCRkCoI8UxD0KY7KU0MFBYEETEGQV+7gUVUQqEdQS6wegYiCILcqv/gLTeUza6oiqKlyZLEqAgmZgiCv4qrTK2tqqEeV01CrIpCQKQjyKq46h46CoEfe2SPQXkMSLgVBXm0TBOoR1BJrakhEQZBb6hGk4rEqAhEFQV6pR5BKXFJFIKIgyCv1CFLx5HUpNikIJFwKgrxSjyCVrbuPampIwqUgyKsuQaAeQU86g6BJQSDhyjQIzGySma0xs7VmNqeHdf6rmT1pZqvM7BdZjiconc1i9Qh60zk1pIpAApbZxKiZFYHrgOOBNqDFzBa5+5NV64wD/gk4wt1fMbO9shpPcNQjSGVrRaAegYQry4rgUGCtuz/j7luA+cDUbuucD1zn7q8AuPvLGY4nLOoRpFKpCJpUEUjAsgyCUcALVffbkmXV9gH2MbOHzOwRM5tU64nMbLqZLTOzZe3t7RkNN2fUI0gnVkUgkmUQWI1l3u1+EzAO+ARwOvBTMxu6zRe5z3X3ie4+cfjw4XUfaC51OaBMU0M9qUwNFZsGNngkIo2TZRC0AWOq7o8G1tdY515373D3Z4E1lINBdpYOKEtHxxGIZBoELcA4M2s2s4HANGBRt3XuAT4JYGbDKE8VPZPhmMKhZnEqnlROTdp9VAKWWRC4ewmYBdwHrAbucvdVZnaZmU1JVrsP2GBmTwJLgP/m7huyGlNQtukRqFlcU3KuoaKCQAKWaT3s7ouBxd2WfavqtgOXJv+kntQjSCd5nYo615AELFVFYGaXpFkm/Yh6BOnEJSI3CsVio0ci0jBpp4bOrrHsnDqOQ+pNPYJ04hIRCgEJW6/1sJmdDpwBNJtZdaN3MKC5/P5MPYJULI4oUUQ7j0rI+poYXQq8CAwDfly1/HVgRVaDkjpQRZBOXCLSuRclcL0Ggbs/BzwHfOydGY7UjU46l4rFJSLT1JCELdWuEmb2OluPCh4IDADedPchWQ1MdpIqgnQ8Uo9AgpcqCNx9cPV9M/s05ZPKSX+lk86lYmoWi+zY5Ki73wMcW+exSD3ppHOpmEfECgIJXNqpoc9U3S0AE9n2BHLSn+iAslTKPQI1iyVsaQ+n/Meq2yVgHdteW0D6k1oHlLmD1TopbLhUEYik7xF8MeuBSJ117xEAeAzaQ6YLc+01JJL2FBMfNLN/N7N2M3vZzO41sw9mPTjZCd17BNXLpJPFqghE0k6O/gK4CxgBjAQWAHdkNSipg+49AlAQ1GAeEasikMClDQJz99vcvZT8+zlqFvdv3XsE1cukU0FBIJK6WbzEzOZQvgC9A6cB/8vM9gBw940ZjU92VFwq9wPMqoJAxxJ0Z14iNp2CWsKW9hNwWvL/Bd2Wn0s5GNQv6G/i0tYAUI+gR6oIRNIHwb7uvrl6gZkN6r5M+pEuQaCpoZ4UPCIq6OpkEra0PYKlKZdJfxFHCoIUVBGI9H09gvcDo4D3mNmBQOVopCHArhmPTXZGXNo6JaQg6FGBCFcQSOD6mho6kfKVyEYDV1Qtfx34ekZjknqoOTWkZnF35YpAzWIJW1/XI7gFuMXMPuvuv3yHxiT1oGZxKgVXRSCS9k+h/czsI90XuvtldR6P1It6BKloakgkfRC8UXV7EHAysLr+w5G6iUtQSPYFMFUEPSmqIhBJfdK56usVY2Y/Ahb1sLr0B+oRpFIgwgsKAgnbjp6IfVd0EFn/ph5BKuWKQM1iCVvaC9M8wdZzCxWAvYDvZjUoqQOPVRGkUCDeGpQigUr7p9DJwPuAo4ChwGJ3b81sVLLzdBxBKkU1i0VSTw1NBW4DhgEDgJvM7KLMRiU7T6eYSKVI1d5VIoFK+wk4Dzjc3d8EMLMfAA8D12Y1MNlJahanUvQYVxBI4FJfjwCo/i0SsfV0E9IfdTmOQM3inhSJdPlOCV7aP4VuAh41s7uT+58GbsxmSFIXcQmKyVk1NTXUoyKRKgIJXqqKwN2vAL4IbAReAb7o7lf19XVmNsnM1pjZ2uTCNj2td6qZuZlNTDtw6YN6BKk0EWMKAglc6k+Au/8Z+HPa9c2sCFwHHA+0AS1mtsjdn+y23mDgYuDRtM8tKahH0Kc4iiiYqyKQ4O3oAWVpHAqsdfdn3H0L5ctcTq2x3neBHwK6yE096YCyPpVKHQCqCCR4WQbBKOCFqvttybJOyTUOxrj7r3t7IjObbmbLzGxZe3t7/UeaR3Gk4wj6ECVBoAPKJHRZBkGtvYq880GzAnAl8LW+nsjd57r7RHefOHz48DoOMcfUI+hTpSKgqIpAwpZlELQBY6rujwbWV90fDOwH/N7M1gGHA4vUMK4TBUGf4s6KQEEgYcsyCFqAcWbWbGYDgWlUnbHU3V9z92HuPtbdxwKPAFPcfVmGYwpHzR6BmsXV1CMQKcssCNy9BMwC7qN87YK73H2VmV1mZlOy+r6SUI+gT3GUvB4KAglcpp8Ad18MLO627Fs9rPuJLMcSHE0N9anSLC6oRyCBy3JqSBpJQdCnqKSKQAQUBPmlA8r6FEdbADBVBBI4BUFedTnpXAEwVQTdxElFoCCQ0CkI8qr6wjRQDgUFQRdR0ixWj0BCpyDIq+qpIVAQ1BBHld1HBzR4JCKNpSDII/cegkA9gmqV3Uc1NSShUxDkkcfl/7sEQVEVQTexdh8VARQE+VT5ha8eQa9UEYiUKQjyKK6xf7yCYBuVHkGhqB6BhE1BkEc9BoF6BNVcew2JAAqCfKr8wlePoFeVqaFCkyoCCZuCII/UI0hFFYFImYIgj9QjSMVj9QhEQEGQTwqCVCpTQ0VVBBI4BUEe1QyCoprF3XRODTUpCCRsCoI8qtksVkXQ3dYewcAGj0SksRQEeaRmcTrJ61FURSCBUxDkkXoEqXT2CLT7qAROQZBH6hGkE2v3URFQEOSTegSpePJ6NKkikMApCPJIPYJ0Yh1ZLAIKgnxSjyCdSBWBCCgI8kk9gnSS10PNYgmdgiCPVBGkoh6BSJmCII86m8XqEfTG4hKxG4Vise+VRXJMQZBHqghS8bhESR8BEX0KckkXpknF4hIRqgZEFAR51GOzWBVBFx4pCERQEOSTDihLxeISkekjIKJPQR7pgLJ0NDUkAmQcBGY2yczWmNlaM5tT4/FLzexJM1thZg+Y2QeyHE8w1CNIxTQ1JAJkGARmVgSuAyYD44HTzWx8t9WWAxPdfQKwEPhhVuMJinoE6cQKAhHItiI4FFjr7s+4+xZgPjC1egV3X+Lum5K7jwCjMxxPONQjSMW8RGwKApEsg2AU8ELV/bZkWU++BPym1gNmNt3MlpnZsvb29joOMafUI0jFVBGIANkGgdVY5jVXNDsLmAj8a63H3X2uu09094nDhw+v4xBzqvILv/qv3UIRcIjjhgypPyqoIhABIMsrcrQBY6rujwbWd1/JzI4DvgEc4+5vZziecPTUI6g8VtA1eqHcLI6145xIpp+CFmCcmTWb2UBgGrCoegUzOxD4N2CKu7+c4VjC0tNeQ9WPSTkIVBGIZBcE7l4CZgH3AauBu9x9lZldZmZTktX+FdgNWGBmj5nZoh6eTrZHZfqnVhC4diGtUBCIlGV6sVZ3Xwws7rbsW1W3j8vy+wcrLgEGhaqcV0WwjfLUkIJARBOkeRSXulYDUBUEqggqCqoIRAAFQT7VDIKqZrEACgKRCgVBHsVRLxWBgqBCQSBSpiDIo7jU9WAyUBDUUPASbpm2yUTeFRQEeaQeQSqqCETKFAR5pB5BKgWPcAWBSLa7j0qDqEeQSoEI7z6FJu+ojo4O2tra2Lx5c6OHkhuDBg1i9OjRDBgwIPXXKAjySD2CVIoeqUfQYG1tbQwePJixY8diVuv0ZLI93J0NGzbQ1tZGc3Nz6q/T1FAe9dojUBBUFNDUUKNt3ryZPffcUyFQJ2bGnnvuud0VloIgj3rtEahZXFH0GO/+Osk7TiFQXzvyeioI8kgVQSqqCETKFAR5FEfqEaRQJFJFIO+4devW8Ytf/GK7v+6cc85h4cKFGYxIQZBPqghSKRJ1vXiPyDtgR4MgS/pzKI90QFkqRVdF0J/887+v4sn1f6/rc44fOYRv/+NHel3n5z//Oddccw1btmzhsMMO4+tf/zrHHXccDz/8MHvssQfHHHMM3/zmN9lnn32YNGkShx12GMuXL2efffbh1ltvZdddd6W1tZVLL72UN954g2HDhnHzzTczYsQI1q5dy4UXXkh7ezvFYpEFCxYwZ84cVq9ezQEHHMDZZ5/NxRdfzJw5c/j973/P22+/zcyZM7ngggtwdy666CJ+97vf0dzcjHvNCzzWhSqCPNIBZakUibd9nSQoq1ev5s477+Shhx7iscceo1gs8oc//IHZs2dz4YUX8uMf/5jx48dzwgknALBmzRqmT5/OihUrGDJkCD/5yU/o6OjgoosuYuHChbS2tnLuuefyjW98A4AzzzyTmTNn8vjjj7N06VJGjBjB97//fY466igee+wxvvrVr3LjjTey++6709LSQktLC/PmzePZZ5/l7rvvZs2aNTzxxBPMmzePpUuXZvY66FOQR+oRpNKkA8r6lb7+cs/CAw88QGtrK4cccggAb731FnvttRff+c53WLBgATfccAOPPfZY5/pjxozhiCOOAOCss87immuuYdKkSaxcuZLjjz8egCiKGDFiBK+//jp//etfOeWUU4DygV613H///axYsaJz/v+1117j6aef5o9//COnn346xWKRkSNHcuyxx2b2OigI8iguQdMuXZcpCLZRpMYR2BIUd+fss8/me9/7XpflmzZtoq2tDYA33niDwYMHA9vummlmuDsf+chHePjhh7s89ve/p5vmcneuvfZaTjzxxC7LFy9e/I7tWqupoTxSj6BPcRRRNFcQBO5Tn/oUCxcu5OWXy5dM37hxI8899xyzZ8/mzDPP5LLLLuP888/vXP/555/v/IV/xx13cOSRR/KhD32I9vb2zuUdHR2sWrWKIUOGMHr0aO655x4A3n77bTZt2sTgwYN5/fXXO5/zxBNP5Prrr6ejowOAp556ijfffJOjjz6a+fPnE0URL774IkuWLMnsdVAQ5JF6BH2KouR10NRQ0MaPH8/ll1/OCSecwIQJEzj++ONZt24dLS0tnWEwcOBAbrrpJgD23XdfbrnlFiZMmMDGjRuZMWMGAwcOZOHChcyePZv999+fAw44oHM+/7bbbuOaa65hwoQJfPzjH+ell15iwoQJNDU1sf/++3PllVdy3nnnMX78eA466CD2228/LrjgAkqlEqeccgrjxo3jox/9KDNmzOCYY47J7HWwLDvRWZg4caIvW7as0cPo337ycdijGabdvnXZq8/DVR+FqdfBgWc1bmz9xOZNbzDoh6N4+IMX87EvfLfRwwnW6tWr2XfffRs9jFTWrVvHySefzMqVKxs9lD7Vel3NrNXdJ9ZaXxVBHuk4gj6VSuUy3FQRiCgIcklB0KeoVJkaUo9A0hk7duy7ohrYEQqCPFKzuE9RaQsAVlQQiCgI8qjmhWnULK4WqyIQ6aQgyCNdmKZPpajSI1AQiCgI8kg9gj5VKgJNDYkoCPJJPYI+RaoIJAO77bYbAOvXr+fUU0/tdd2rrrqKTZs2dd4/6aSTePXVVzMdX08UBHlUq0dgyVutigCAuLL7qCoC6UMUbf8fTyNHjuzz2gHdg2Dx4sUMHTp0u79XPehTkEe1egRm5XBQEAAQRwqCfuc3c+ClJ+r7nO//KEz+fo8Pr1u3ruappcePH8+5557L/fffz6xZszjkkEOYOXMm7e3t7LrrrsybN48Pf/jDPPvss5xxxhmUSiUmTZrU5XkrB59FUcTs2bO57777MDPOP/983J3169fzyU9+kmHDhrFkyRLGjh3LsmXLGDZsGFdccQU/+9nPADjvvPP4yle+wrp165g8eTJHHnkkS5cuZdSoUdx777285z3v2emXSRVBHtWaGgIFQZXKcQSaGpJap5aG8tlCH3zwQaZNm8b06dO59tpraW1t5Uc/+hFf/vKXAbjkkkuYMWMGLS0tvP/976/5/HPnzuXZZ59l+fLlrFixgjPPPJOLL76YkSNHsmTJkm3OIdTa2spNN93Eo48+yiOPPMK8efNYvnw5AE8//TQzZ85k1apVDB06lF/+8pd1eQ30KcijXoNAPQKAODnXUKFJH4F+o5e/3LNU69TSAKeddhpQPvvo0qVL+dznPtf5NW+//TYADz30UOcv489//vPMnj17m+f/7W9/y4UXXkhT8rO2xx579DqeBx98kFNOOYX3vve9AHzmM5/hT3/6E1OmTKG5uZkDDjgAgIMPPph169bt6GZ3kemnwMwmAVcDReCn7v79bo/vAtwKHAxsAE5z93VZjin34hjo4ayahaIqgkTn1FBhQINHIo1W69TSQOcv4jiOGTp0aJfrEvT29d25+3adTrq387/tssvW08sXi0Xeeuut1M/bm8ymhsysCFwHTAbGA6eb2fhuq30JeMXd/zNwJfCDrMYTjLiXs2pqaqhTZ0WgHkHwap1autqQIUNobm5mwYIFQPkX9eOPPw7AEUccwfz58wG4/fbbqeWEE07ghhtuoJRMR27cuBFgm9NRVxx99NHcc889bNq0iTfffJO7776bo446qg5b2rMsPwWHAmvd/RkAM5sPTAWerFpnKvCd5PZC4H+amXkGp0Rt+dXVDF85r95P2+8UcPYGfvrQ89y57A9dHpv/VszAljvY0HpfYwbXj7wv3gyoRyBbTy19wQUXMG7cOGbMmMG1117bZZ3bb7+dGTNmcPnll9PR0cG0adPYf//9ufrqqznjjDO4+uqr+exnP1vz+c877zyeeuopJkyYwIABAzj//POZNWsW06dPZ/LkyYwYMaJLn+Cggw7inHPO4dBDD+38+gMPPLBu00C1ZHYaajM7FZjk7ucl9z8PHObus6rWWZms05bc/49knb91e67pwHSAvffe++Dnnntuu8ez/P6f4yvu3NHNeVeJKPKbPb/ASwM/0GX5J19ZyH/anM+TZu2IUtNu7PvF6xi8e+9ztpKdRp+G+t10auntsb2noc7yz6Fak2LdUyfNOrj7XGAulK9HsCODOfCEs+CEcM7Df0jNpQe/w6MQkXeDLHcfbQPGVN0fDazvaR0zawJ2BzZmOCYRkU55PrX09sgyCFqAcWbWbGYDgWnAom7rLALOTm6fCvwui/6AiPRf+sjX1468npkFgbuXgFnAfcBq4C53X2Vml5nZlGS1G4E9zWwtcCkwJ6vxiEj/M2jQIDZs2KAwqBN3Z8OGDQwaNGi7vk7XLBaRhuno6KCtrY3Nmzc3eii5MWjQIEaPHs2AAV2PkWlUs1hEpFcDBgygubm50cMIns41JCISOAWBiEjgFAQiIoF71zWLzawd2P5Di8uGAX/rc638CXG7Q9xmCHO7Q9xm2P7t/oC7D6/1wLsuCHaGmS3rqWueZyFud4jbDGFud4jbDPXdbk0NiYgETkEgIhK40IJgbqMH0CAhbneI2wxhbneI2wx13O6gegQiIrKt0CoCERHpRkEgIhK4YILAzCaZ2RozW2tmuTzLqZmNMbMlZrbazFaZ2SXJ8j3M7P+Y2dPJ/+9r9FjrzcyKZrbczH6d3G82s0eTbb4zORV6rpjZUDNbaGZ/Sd7zjwXyXn81+fleaWZ3mNmgvL3fZvYzM3s5uYpjZVnN99bKrkl+t60ws4O29/sFEQRmVgSuAyYD44HTzWx8Y0eViRLwNXffFzgcmJls5xzgAXcfBzxAPk/3fQnl051X/AC4MtnmV4AvNWRU2boa+N/u/mFgf8rbn+v32sxGARcDE919P6BI+VoneXu/bwYmdVvW03s7GRiX/JsOXL+93yyIIAAOBda6+zPuvgWYD0xt8Jjqzt1fdPc/J7dfp/yLYRTlbb0lWe0W4NONGWE2zGw08F+Anyb3DTgWWJisksdtHgIcTfmaHrj7Fnd/lZy/14km4D3JVQ13BV4kZ++3u/+Rba/W2NN7OxW41cseAYaa2Yjt+X6hBMEo4IWq+23Jstwys7HAgcCjwD+4+4tQDgtgr8aNLBNXAf8diJP7ewKvJhdHgny+3x8E2oGbkimxn5rZe8n5e+3ufwV+BDxPOQBeA1rJ//sNPb+3O/37LZQgsBrLcrvfrJntBvwS+Iq7/73R48mSmZ0MvOzurdWLa6yat/e7CTgIuN7dDwTeJGfTQLUk8+JTgWZgJPBeylMj3eXt/e7NTv+8hxIEbcCYqvujgfUNGkumzGwA5RC43d1/lSz+f5VSMfn/5UaNLwNHAFPMbB3lKb9jKVcIQ5OpA8jn+90GtLn7o8n9hZSDIc/vNcBxwLPu3u7uHcCvgI+T//cben5vd/r3WyhB0AKMS/YsGEi5ubSowWOqu2Ru/EZgtbtfUfXQIuDs5PbZwL3v9Niy4u7/5O6j3X0s5ff1d+5+JrAEODVZLVfbDODuLwEvmNmHkkWfAp4kx+914nngcDPbNfl5r2x3rt/vRE/v7SLgC8neQ4cDr1WmkFJz9yD+AScBTwH/AXyj0ePJaBuPpFwSrgAeS/6dRHnO/AHg6eT/PRo91oy2/xPAr5PbHwT+L7AWWADs0ujxZbC9BwDLkvf7HuB9IbzXwD8Df6HEc0MAAABZSURBVAFWArcBu+Tt/QbuoNwD6aD8F/+XenpvKU8NXZf8bnuC8h5V2/X9dIoJEZHAhTI1JCIiPVAQiIgETkEgIhI4BYGISOAUBCIigVMQiIgETkEgIhK4/w9eTWTAzys7kgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reg_score = np.sqrt(metrics.mean_squared_error(pred,y_true))\n",
    "print(\"Score (RMSE): {}\".format(reg_score))\n",
    "\n",
    "# Plot the chart\n",
    "chart_regression(pred, y_true, sort=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dense Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_prep = pd.read_csv(r'data/nid_test_prep.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "y=X_test_prep['outcome_normal.']\n",
    "X_test_prep=X_test_prep.drop('outcome_normal.', axis=1)\n",
    "X_numpy = X_test_prep.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split into train/test\n",
    "x_train, x_test, y_train, y_test = train_test_split(X_numpy, y, test_size=0.25, random_state=42)\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "y_train = tf.keras.utils.to_categorical(y_train, 2)\n",
    "y_test = tf.keras.utils.to_categorical(y_test, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 86631 samples, validate on 28877 samples\n",
      "Epoch 1/100\n",
      "86631/86631 - 7s - loss: 0.0558 - acc: 0.9816 - val_loss: 0.0352 - val_acc: 0.9908\n",
      "Epoch 2/100\n",
      "86631/86631 - 6s - loss: 0.0195 - acc: 0.9940 - val_loss: 0.0203 - val_acc: 0.9935\n",
      "Epoch 3/100\n",
      "86631/86631 - 6s - loss: 0.0149 - acc: 0.9953 - val_loss: 0.0207 - val_acc: 0.9922\n",
      "Epoch 4/100\n",
      "86631/86631 - 7s - loss: 0.0119 - acc: 0.9964 - val_loss: 0.0169 - val_acc: 0.9939\n",
      "Epoch 5/100\n",
      "86631/86631 - 6s - loss: 0.0106 - acc: 0.9969 - val_loss: 0.0224 - val_acc: 0.9927\n",
      "Epoch 6/100\n",
      "86631/86631 - 7s - loss: 0.0109 - acc: 0.9968 - val_loss: 0.0107 - val_acc: 0.9976\n",
      "Epoch 7/100\n",
      "86631/86631 - 6s - loss: 0.0093 - acc: 0.9972 - val_loss: 0.0128 - val_acc: 0.9966\n",
      "Epoch 8/100\n",
      "86631/86631 - 6s - loss: 0.0088 - acc: 0.9974 - val_loss: 0.0083 - val_acc: 0.9979\n",
      "Epoch 9/100\n",
      "86631/86631 - 6s - loss: 0.0091 - acc: 0.9973 - val_loss: 0.0089 - val_acc: 0.9979\n",
      "Epoch 10/100\n",
      "86631/86631 - 6s - loss: 0.0087 - acc: 0.9975 - val_loss: 0.0084 - val_acc: 0.9978\n",
      "Epoch 11/100\n",
      "86631/86631 - 6s - loss: 0.0078 - acc: 0.9976 - val_loss: 0.0097 - val_acc: 0.9967\n",
      "Epoch 00011: early stopping\n",
      "Model: Sequential, Activation: tanh, Layers, count: 32, 6, 2  Optimizer: adam \n",
      "Elapsed time: 0:07:16.35\n"
     ]
    }
   ],
   "source": [
    "# Defining a dense model\n",
    "model = Sequential()\n",
    "model.add(Dense(32, input_dim=X_numpy.shape[1], activation='tanh'))\n",
    "model.add(Dense(6))\n",
    "model.add(Dense(2,activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "monitor = EarlyStopping(monitor='val_loss', min_delta=1e-6, patience=3, verbose=1, mode='auto')\n",
    "checkpointer = ModelCheckpoint(filepath=\"data/best_fullyconnected_weights.hdf5\", verbose=0, save_best_only=True) # save best model\n",
    "\n",
    "model.fit(x_train,\n",
    "          y_train,\n",
    "          validation_data=(x_test,y_test),\n",
    "          callbacks=[monitor,checkpointer],\n",
    "          verbose=2,epochs=100)\n",
    "\n",
    "model.load_weights('data/best_fullyconnected_weights.hdf5') # load weights from best model\n",
    "\n",
    "\n",
    "elapsed_time = time.time() - start_time\n",
    "print(\"Model: Sequential, Activation: tanh, Layers, count: 32, 6, 2  Optimizer: adam \")\n",
    "print(\"Elapsed time: {}\".format(hms_string(elapsed_time)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100/100 - 0s - loss: 0.0500 - acc: 0.9900\n"
     ]
    }
   ],
   "source": [
    "# evaluate() computes the loss and accuracy\n",
    "score = model.evaluate(x_test[0:100], y_test[0:100], verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_22\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_48 (Dense)             (None, 32)                3808      \n",
      "_________________________________________________________________\n",
      "dense_49 (Dense)             (None, 6)                 198       \n",
      "_________________________________________________________________\n",
      "dense_50 (Dense)             (None, 2)                 14        \n",
      "=================================================================\n",
      "Total params: 4,020\n",
      "Trainable params: 4,020\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.050018357187509535\n",
      "Test accuracy: 0.9900000095367432\n"
     ]
    }
   ],
   "source": [
    "print('Test loss: {}'.format(score[0]))\n",
    "print('Test accuracy: {}'.format(score[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.99\n",
      "Averaged F1: 0.9899867765232429\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.98      0.99        44\n",
      "           1       0.98      1.00      0.99        56\n",
      "\n",
      "    accuracy                           0.99       100\n",
      "   macro avg       0.99      0.99      0.99       100\n",
      "weighted avg       0.99      0.99      0.99       100\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_true = np.argmax(y_test[0:100],axis=1)\n",
    "pred = model.predict(x_test[0:100])\n",
    "pred = np.argmax(pred,axis=1)\n",
    "\n",
    "\n",
    "score = metrics.accuracy_score(y_true, pred)\n",
    "print('Accuracy: {}'.format(score))\n",
    "\n",
    "\n",
    "f1 = metrics.f1_score(y_true, pred, average='weighted')\n",
    "print('Averaged F1: {}'.format(f1))\n",
    "\n",
    "           \n",
    "print(metrics.classification_report(y_true, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZxN9f/A8dfbvgvTItTYMmNfJtmSkiUJSUlEIqEskVTapSQhX3zlJ6lUpLKUFpTlW5FG9l0qBmWJsTPG+/fHOTNdY+bONblz7sy8n4/Hfcw9+/t+5tzzvufzOedzRFUxxhhjUpLN6wCMMcaENksUxhhj/LJEYYwxxi9LFMYYY/yyRGGMMcYvSxTGGGP8skSRyYlIRxGZ73UcoUREjolIGQ+2Gy4iKiI50nvbwSAiG0SkURqWS/M+KSJNRWR2WpZNKxHJLSKbReSK9NxuKLFEkY5E5HcROekeqP4UkakiUiCY21TVD1S1aTC34UtE6onIdyJyVERiReRzEamYXttPJp7FItLdd5yqFlDVHUHa3nUiMlNEDriff62IDBCR7MHYXlq5Cavcv1mHqlZS1cWpbOeC5Pgv98lXgOE+61cROe5+p3aLyKikZS0iLUVkhTvfQRH5QERKJpmnuIi8LSJ73X13s4i8KCL5VfU0MAUYnMaYMzxLFOnvDlUtAFQHagBPeRxPmiT3q1hE6gLzgTnA1UBpYA3wQzB+wYfaL3MRKQv8BOwCqqhqYeBuIAooeIm35dln92rbInI9UFhVlyeZVM39Tt0EtAce9FmmHfAh8CYQBlQCTgPfi0gRd56iwDIgL1BXVQsCTYDLgLLuqj4EuohI7iB9vNCmqvZKpxfwO3Crz/AIYJ7PcG5gJLAT+AuYCOT1md4aWA0cAX4FmrvjCwNvA3uB3cDLQHZ32gPA9+77icDIJDHNAQa4768GPgX2A78BfX3mewH4BJjmbr97Mp/vf8CEZMZ/Bbznvm8ExABPAwfcMukYSBn4LDsY+BN4HygCfOHGfMh9X9KdfxgQD5wCjgHj3PEKlHPfTwXGA/OAozgH+rI+8TQFtgCxwARgSXKf3Z13mu//M5np4e62u7if7wAwxGd6bZwD1mH3fzkOyOUzXYFHgG3Ab+64N3ES0xFgJXCjz/zZ3XL+1f1sK4FSwFJ3Xcfdcmnvzt8SZ/86DPwIVE2y7w4G1uIcaHPgsz+7sUe7cfwFjHLH73S3dcx91cVnn3TnqQQsAP52l306hfJ7DpicZFzi/9Id/hgY774X4A/giSTLZAPWAy+5wy8D64BsqXx/twE3eX0c8eLleQBZ6ZXki1XS3Tnf9Jk+BpgLFMX5Bfo58Ko7rbZ7sGri7uglgAh32mzgLSA/cAWwAnjYnZb4pQQaugcVcYeLACdxEkQ290DyHJALKAPsAJq5874AxAFt3HnzJvls+XAOyjcn87m7Anvd942As8AonKRwE84Bq0IAZZCw7GvusnmBYsBd7vYLAjOB2T7bXkySAzsXJoq/3fLNAXwATHenheEc+Nq60/q5ZZBSovgT6Orn/x/ubvv/3Nir4Rx0I93ptYA67rbCgU1A/yRxL3DLJiF5dnLLIAcw0I0hjzttEM4+VgHnoFkNKJa0DNzhmsA+4AacBNMFZ3/N7bPvrsZJNHl9xiXsz8uA+933BYA6ST5zDp9tPcA/+2RBnKQ4EMjjDt+QQvnNBAb5+V9GuOt6zGdYgdLJrOtFYJn7fjnwYgDf37n4/HjKSi/PA8hKL/eLdQzn150C3wKXudME54Dp+2u2Lv/8cnwLGJ3MOq90Dza+Zx4dgEXue98vpeD8wmvoDj8EfOe+vwHYmWTdTwHvuO9fAJb6+Wwl3c8Ukcy05kCc+74RzsE+v8/0j4FnAyiDRsAZ3ANhCnFUBw75DC8m9UQx2WdaC2Cz+75zwsHEp/x2JV2fz/Q43LO8FKaHu9su6TNuBXBvCvP3B2YlifuWVPaxQzhVMeCcCbVOYb6kieK/wNAk82zB/QXt7rsPJrM/JySKpTgH37AUPnNKiaIDsCrA788CoGcyn+OIu98o8BH/JLcG7rgL9hegJ7DNfb8t6XpT2P4HwHOBxJrZXtZGkf7aqFMH2gjnF0+YO/5ynF/FK0XksIgcBr52x4PzS+7XZNZ3LZAT2Ouz3Fs4ZxbnUWdvn47z5QS4D2fnT1jP1QnrcNfzNE4iSrDLz+c6BJwDiiczrThONUvivKp63Gf4D5yzmtTKAGC/qp5KGBCRfCLyloj8ISJHcA5Yl11k4/GfPu9P4Pwixo0p8TO75RfjZz0HSf7zB7Q9tyH8C/dChyM4DbdhSZY9738gIgNFZJPbcH4YpxoyYZmU9pnkXAsMTPL/L4VTBsluO4luwHXAZhH5WURaBrjdi4nxEMm39dTEKcP2OD948rvjE/a51PbJQP9vBXGq5bIcSxQeUdUlOL9mR7qjDuBUA1VS1cvcV2F1GunA+ZKWvXBN7MI5owjzWa6QqlZKYdMfAe1E5FqcL9WnPuv5zWcdl6lqQVVt4Ru2n89zHKf64e5kJt+Dc/aUoIiI5PcZvgbYE0AZJBfDQJyqlRtUtRBO9Ro4v/79xhyAvThnSs4KRcR3OBkLcarB0uq/wGagvPtZnuafz5Eg8fOIyI047Qb3AEVU9TKc6smEZVLaZ5KzCxiW5P+fT1U/Sm7bSanqNlXtgPMD5TXgE/d/nFr5X0yMa3GSUXLbV1X9GGcffM4dvQUnsZ+3T4pINpz/U8I+uRC40x3vTyTOxRlZjiUKb40BmohIdVU9h1N3PTrhem0RKSEizdx53wa6ikhjEcnmTotQ1b04Vxq9ISKF3GllReSm5DaoqqtwGn4nA9+oasIvpBXAEREZLCJ5RSS7iFR2rzQJ1JM4V4b0FZGCIlJERF7GqT56Mcm8L4pILvdg1xKYGUAZJKcgTnI57F698nyS6X/htLekxTygioi0ca/0eQS4ys/8zwP1ROR1EbnKjb+ciEwTkcsC2F5BnGqUYyISAfQKYP6zOP/PHCLyHFDIZ/pkYKiIlBdHVREp5k5LWi7/B/QUkRvcefOLyO0iEtDVWiLSSUQud/+HCftUvBvbOVL+H3wBXCUi/cW5X6GgiNyQwrxf4rRp+TMc6CEiV7lngI8Dz4jIfe5+fRVOuRQCRrvLjHKH33V/QCXsd6NEpGrCME7bUNIrrrIESxQeUtX9wHs49fPg/DrcDix3qx4W4vxaRlVX4DQKj8b51bgEp7oAnLr0XMBGnNPzT/B/Kv0RcCvOJX8JscQDd+DU8f+G8+t+Mk5VRqCf53ugGU7j716cKqUaQANV3eYz659unHtwqr56qurm1MogBWNwGoYP4HyJv04y/U2cM6hDIjI20M/ifp4DOL9GR+BUT1TEubLndArz/4qTFMOBDSISi3PGFo3TLpWax3GqA4/iHLhnpDL/NzhXlG3FKetTnF89NAqn/Wc+TgJ6G6eswGlzetetZrpHVaNx2qzG4fxvtuO0JQSqOc5nPoZT5veq6ilVPYFz9dkP7rbq+C6kqkdxLtC4A2e/2AbcnNwGVPUXINZPIkFV1+F8Nwa5wzOA+4HHcPaRjW4Z1FfVg+48fwP1cNqYfhKRozhnG7FuOYDzf3lXnXsqspyEq1+MSRfi3Mk7TVX9VeGEJLdqIgbnct5FXseTFYlIU6C3qrZJx23mxqlyaqiq+9Jru6EkpG5YMibUuNVeP+FUbw3Cqf/PktUPoUBV5+OcIaXnNk/jXHiSZVnVkzH+1cW5KucATvVIG1U96W1IxqQvq3oyxhjjl51RGGOM8SvDtVGEhYVpeHi412EYY0yGsnLlygOqennqc14owyWK8PBwoqOjvQ7DGGMyFBH5I63LWtWTMcYYvyxRGGOM8csShTHGGL8sURhjjPHLEoUxxhi/LFEYY4zxK2iJQkSmiMg+EVmfwnQRkbEisl1E1opIzWDFYowxJu2CeUYxFafr4ZTcBpR3Xz1wHtpijDEmxATthjtVXSoi4X5maQ285z5cZLmIXCYixd0H8aTJhz/tZM7q3Wld3BhjMhVVZffqJexeveRfrcfLO7NLcP5DVmLccRckChHpgXPWwTXXXJPiCues3s3GvUeoWLxQivMYY0xWcPzAXn6Z8QZ71/1I4RLl/tW6vEwUSZ8FDCk8X1dVJwGTAKKiovx2d1uxeCFmPFz330dnjDEZlKoSFRXFkR1beOONN+jbty85c+ZM8/q8TBQxQCmf4ZI4j8b0a8f+47R/a1my0+xswhiTlf34449UqVKFggULMnnyZMLCwihVqlTqC6bCy8tj5wKd3auf6gCxgbRPnIyLT3FaxeKFaF29xCUM0RhjQt/Bgwd56KGHqF+/Pm+88QYANWrUuCRJAoJ4RiEiHwGNgDARiQGeB3ICqOpE4EugBc7Dy08AXQNZb96c2a1qyRhjcKqY3nvvPR5//HEOHTrEoEGDGDRo0CXfTjCveuqQynQFHgnW9o0xJrMbPHgwr7/+OvXq1WPixIlUqVIlKNvJcM+jMMaYrOzkyZMcP36csLAwunXrRvny5enWrRvZsgWvJcG68DDGmAzi66+/pnLlyjz88MMAVKhQgYceeiioSQIsURhjTMjbs2cP99xzD7fddhs5c+bk0UcfTdftW9WTMcaEsG+//ZY777yTM2fOMHToUAYNGkTu3LnTNQZLFMYYE4Li4uLImTMn1apVo0WLFrz88suUK/fv7rBOK6t6MsaYEHLkyBH69evHjTfeSHx8PGFhYUyfPt2zJAGWKIwxJiSoKjNnziQiIoL//Oc/REVFcfr0aa/DAqzqyRhjPLd//366dOnCV199RY0aNZgzZw7XX3+912ElsjMKY4zxWKFChThw4ABjxoxhxYoVIZUkwBKFMcZ4YunSpTRr1oxjx46RO3duli9fTr9+/ciRI/QqeixRGGNMOjpw4ABdu3blpptuYuvWrfz+++8AQb9p7t8I3ciMMSYTUVWmTJlChQoVmDZtGk899RQbNmygcuXKXoeWqtA7xzHGmExq2rRpVKxYkYkTJ1KpUiWvwwmYnVEYY0yQnDhxgmeeeYaYmBhEhE8//ZQlS5ZkqCQBliiMMSYovvzySypVqsSwYcP4/PPPAShSpEhIt0WkJONFbIwxISwmJoZ27dpx++23kzdvXpYsWUKvXr28DutfsURhjDGX0LBhw5g3bx6vvPIKq1evpmHDhl6H9K+J86C5jKPotZH69x+bvA7DGGMSrVixgrx581KlShUOHjxIbGwsZcqU8Tqs84jISlWNSsuydkZhjDFpFBsbyyOPPEKdOnUYMmQIAMWKFQu5JPFvWaIwxpiLpKpMnz6diIgIJk6cSJ8+fZg2bZrXYQWN3UdhjDEXadq0aXTu3JmoqCi++OILatWq5XVIQWWJwhhjAnD69Gl27NhBZGQk99xzD2fPnqVz585kz57d69CCzqqejDEmFYsWLaJatWo0a9aM06dPkzt3brp27ZolkgRYojDGmBTt27ePzp07c8sttxAXF8ekSZPS/XnVocCqnowxJhnbt2+ndu3aHDt2jCFDhjBkyBDy5s3rdViesERhjDE+jhw5QqFChShbtizdunXjwQcfJDIy0uuwPGVVT8YYAxw/fpzBgwcTHh6e2Inf66+/nuWTBNgZhTHG8Pnnn/Poo4+yc+dOunXrRr58+bwOKaRYojDGZFlnz57lnnvuYdasWVSqVIn//e9/NGjQwOuwQo5VPRljspyEPu5y5MhB8eLFGT58OL/88osliRRYojDGZCnLly8nKiqKX375BYDx48czePBgcuXK5XFkocsShTEmSzh06BC9evWiXr16/PXXXxw6dMjrkDKMoCYKEWkuIltEZLuIPJnM9GtEZJGIrBKRtSLSIpjxGGOyphkzZhAREcGkSZPo378/mzZtonHjxl6HlWEErTFbRLID44EmQAzws4jMVdWNPrM9A3ysqv8VkYrAl0B4sGIyxmRNmzdvJjw8nK+//poaNWp4HU6GE8wzitrAdlXdoapngOlA6yTzKFDIfV8Y2BPEeIwxWcSpU6d48cUXE59V/fTTT/Pjjz9akkijYCaKEsAun+EYd5yvF4BOIhKDczbRJ7kViUgPEYkWkei4uLhgxGqMySQWLlxI1apVeeGFF1iyZAkAOXPmzDId+AVDMBOFJDMu6XNXOwBTVbUk0AJ4X0QuiElVJ6lqlKpG5cyZMwihGmMyur/++ouOHTvSpEkTVJX58+czcuRIr8PKFIKZKGKAUj7DJbmwaqkb8DGAqi4D8gBhQYzJGJNJLViwgE8++YTnnnuOdevW0aRJE69DyjSCeWf2z0B5ESkN7AbuBe5LMs9OoDEwVUQicRLF/iDGZIzJRNasWcO2bdto164dHTt2pH79+pQuXdrrsDKdoJ1RqOpZ4FHgG2ATztVNG0TkJRFp5c42EHhIRNYAHwEPaMItk8YYk4Jjx44xcOBAatWqxZNPPsnZs2cREUsSQSIZ7bhc9NpI/fuPTV6HYYzxyOzZs+nTpw8xMTH06NGDV199laJFi3odVsgTkZWqGpWWZa1TQGNMhrFu3TruvPNOqlSpwowZM6hXr57XIWUJ1oWHMSakxcXF8d133wFQpUoV5s2bx8qVKy1JpCNLFMaYkPXjjz9Sq1YtmjRpwvbt2wFo0aIFdpl8+rJEYYwJOX///Tc9evSgfv36HD58mM8++4xy5cp5HVaWZW0UxpiQcurUKapXr86ePXsYOHAgL7zwAgUKFPA6rCzNEoUxJiTExMRQsmRJ8uTJw9ChQ6levTrVqlXzOiyDVT0ZYzx28uRJnnvuOcqWLZvYiV+XLl0sSYSQgM4oRCQXcI2qbg9yPMaYLGT+/Pn07t2bX3/9lU6dOlG7dm2vQzLJSPWMQkRuB9YBC9zh6iIyK9iBGWMytz59+tCsWTOyZcvGwoULef/997nyyiu9DsskI5AzipeAG4BFAKq6WkTs8gNjzEWLj48HIHv27NSpU4ewsDAGDx5Mnjx5PI7M+BNIG0Wcqh5OMi5j9fthjPHcL7/8Qt26dZkwYQIAHTt25Pnnn7ckkQEEkig2icg9QDYRKS0iY4DlQY7LGJNJHD16lMcee4zrr7+enTt3Urx4ca9DMhcpkETxKFALOAd8BpwC+gUzKGNM5jB//nwiIyN58803efjhh9m8eTPt2rXzOixzkQJpo2imqoOBwQkjRKQtTtIwxpgU5cqViyuuuIJPP/2UG264wetwTBql2s24iPyiqjWTjFupqrWCGlkKrJtxY0JXXFwco0aN4siRIwwbNgyAc+fOkS2b3bLltaB0My4izYDmQAkRGeUzqRBONZQxxiT6/vvv6dmzJxs2bODuu+9OTBCWJDI+f//BfcB6nDaJDT6v+cBtwQ/NGJMRHDx4kO7du3PjjTdy9OhRPv/8cz7++GNLEJlIimcUqroKWCUiH6jqqXSMyRiTgRw8eJDp06fzxBNP8Nxzz5E/f36vQzKXWCCN2SVEZBhQEUi84FlVrwtaVMaYkLZp0yY+/vhjnn/+ea677jp27txpjyPNxAI5N5wKvAMITpXTx8D0IMZkjAlRJ06cYMiQIVSrVo0333yTmJgYAEsSmVwgiSKfqn4DoKq/quozwM3BDcsYE2q+/vprKleuzCuvvMJ9993Hli1bKFmypNdhmXQQSNXTaRER4FcR6QnsBq4IbljGmFBy7Ngx7r//fooVK8aiRYto1KiR1yGZdBTIGcVjQAGgL1AfeAh4MJhBGWO8Fx8fz7Rp04iPj6dAgQIsXLiQNWvWWJLIglI9o1DVn9y3R4H7AUTEzjeNycRWrlzJww8/zMqVK8mbNy933XWXPUgoC/N7RiEi14tIGxEJc4crich7WKeAxmRKsbGx9O3bl9q1a7N7926mT59O27ZtvQ7LeCzFRCEirwIfAB2Br0VkCM4zKdYAdmmsMZnQXXfdxbhx4+jduzebN2+mffv2OE2UJivzV/XUGqimqidFpCiwxx3ekj6hGWPSw44dO7j88sspWLAgw4YNI1u2bFx//fVeh2VCiL+qp1OqehJAVf8GNluSMCbzOHPmDK+88gqVKlXi5ZdfBuCGG26wJGEu4O+MooyIJHQlLkC4zzCqahWXxmRQS5cupWfPnmzatIl27drRt29fr0MyIcxforgryfC4YAZijEkfo0ePZsCAAYSHhzNv3jxatGjhdUgmxPnrFPDb9AzEGBM8586d4/jx4xQsWJDbb7+d/fv388wzz5AvXz6vQzMZQKoPLgo19uAiYy7Ohg0b6NmzZ+KT5kzW9G8eXBTUDuNFpLmIbBGR7SLyZArz3CMiG0Vkg4h8GMx4jMlKTpw4wVNPPUX16tXZtGkTLVu2JKP9MDShIZC+ngAQkdyqevoi5s8OjAeaADHAzyIyV1U3+sxTHngKqK+qh0TE+pAy5hJYtWoVbdu25ffff6dr166MGDGCsLAwr8MyGVSqZxQiUltE1gHb3OFqIvKfANZdG9iuqjtU9QxO1+Stk8zzEDBeVQ8BqOq+i4reGHOehDOGa665hmuuuYYlS5YwZcoUSxLmXwmk6mks0BI4CKCqawism/ESwC6f4Rh3nK/rgOtE5AcRWS4izQNYrzEmibNnzzJmzBgaN25MfHw8xYoVY8mSJTRs2NDr0EwmEEiiyKaqfyQZFx/Acsnd95+0gjQHUB5oBHQAJovIZResSKSHiESLSHRcXFwAmzYm61ixYgW1a9fmscceI0+ePBw5csTrkEwmE0ii2CUitQEVkewi0h/YGsByMUApn+GSON2AJJ1njqrGqepvwBacxHEeVZ2kqlGqGpUzZ84ANm1M5nfs2DEeeeQR6tSpw19//cXMmTOZN28eRYoU8To0k8kEkih6AQOAa4C/gDruuNT8DJQXkdIikgu4F5ibZJ7ZuNVYbg+11wE7AgvdmKwtZ86cLF68mD59+iTeYW0d+JlgCOSqp7Oqeu/FrlhVz4rIo8A3QHZgiqpuEJGXgGhVnetOayoiG3Gqswap6sGL3ZYxWcX27dt56aWXGD9+PAULFmTlypXkyZPH67BMJpfqDXci8itOldAM4DNVPZoegaXEbrgzWdHp06cZMWIEw4YNI1euXMybN48bb7zR67BMBhLUG+5UtSzwMlALWCcis0Xkos8wjDFps2jRIqpVq8Zzzz1HmzZt2Lx5syUJk64CujNbVX9U1b5ATeAIzgONjDFBpqoMGzaMuLg4vv76a6ZPn87VV1/tdVgmi0m1jUJECuDcKHcvEAnMAeoFOS5jsqxz587x9ttv07x5c0qVKsX777/PZZddRt68eb0OzWRRgZxRrMe50mmEqpZT1YGq+lOQ4zImS1q7di0NGjSgR48eTJ48GYDixYtbkjCeCuSqpzKqei7okRiThR07dowXX3yR0aNHU6RIEaZOnUrnzp29DssYwE+iEJE3VHUg8KmIXHBplD3hzphL54UXXuCNN96ge/fuDB8+nGLFinkdkjGJUrw8VkRqq+oKEWmc3HSvHmxkl8eazGLXrl0cP36ciIgIDhw4wObNm2nQoIHXYZlMKiiXx6rqCvdtpKp+6/vCadQ2xqTB2bNnGTVqFJGRkTz88MMAhIWFWZIwISuQxuwHkxnX7VIHYkxWsHz5cqKiohg4cCCNGjXi3Xff9TokY1Llr42iPc4lsaVF5DOfSQWBw8EOzJjMZt68edxxxx1cffXVfPbZZ7Rp08b6ZjIZgr+rnlbgPIOiJM6T6hIcBVYFMyhjMgtVZc+ePZQoUYJbb72Vl156iX79+lGwYEGvQzMmYKn29RRqrDHbZBRbt26ld+/ebN26lY0bN1KgQAGvQzJZWFAas0Vkifv3kIj87fM6JCJ/pzVYYzK7U6dO8cILL1ClShWio6N56qmn7IY5k6H5q3pKeNypPWzXmAD9+eefNGzYkG3bttGhQwdGjRrFVVdd5XVYxvwr/i6PTbgbuxSQXVXjgbrAw0D+dIjNmAwj4RG9V155JQ0bNmT+/Pl8+OGHliRMphDI5bGzcR6DWhZ4D+ceig+DGpUxGcS5c+eYOHEiZcuWJSYmBhFh8uTJNGnSxOvQjLlkAkkU51Q1DmgLjFHVPkCJ4IZlTOhbs2YN9erVo1evXpQvXz7xrMKYzCaQRHFWRO4G7ge+cMflDF5IxoQ2VeXxxx+nVq1a7Nixg/fff5+FCxdSunRpr0MzJigCvTP7ZpxuxneISGngo+CGZUzoEhEOHTpEt27d2LJlC506dbIb50ymFtB9FCKSAyjnDm5X1bNBjcoPu4/CeOGPP/6gX79+PPfcc9SsWZNz586RLVtAD4g0JiQE9ZnZInIjsB14G5gCbBWR+mnZmDEZTVxcHCNGjKBixYosWLCALVu2AFiSMFlKIA8uGg20UNWNACISCbwPpCkzGZNR/Pjjjzz88MOsX7+e1q1bM3bsWK655hqvwzIm3QWSKHIlJAkAVd0kIrmCGJMxIWHhwoXExsYye/ZsWrdu7XU4xngm1TYKEZkKnMY5iwDoCORT1S7BDS151kZhgkVVef/997n88su57bbbOH36NHFxcdZHk8kUgtpGAfQEfgWeAAYDO3DuzjYm09i8eTO33HILXbp04Z133gEgd+7cliSMIZWqJxGpApQFZqnqiPQJyZj0c/LkSV555RVee+018ufPz1tvvUX37t29DsuYkOKv99incbrv6AgsEJHknnRnTIb2+eef8/LLL9O+fXs2b95Mjx497IomY5Lwd0bREaiqqsdF5HLgS5zLY43J0P78809Wr15N8+bNufvuuwkPD6d27dpeh2VMyPL30+m0qh4HUNX9qcxrTMiLj49nwoQJVKhQgfvvv5+TJ08iIpYkjEmFvzOKMj7PyhagrO+zs1W1bVAjM+YS+uWXX+jZsyc///wzt956KxMmTLCHCRkTIH+J4q4kw+OCGYgxwfLbb79Ru3ZtwsLC+PDDD7n33nutbyZjLkKKiUJVv03PQIy5lFSVdevWUbVqVUqXLs0777zDHXfcwWWXXeZ1aMZkONbuYDKd3377jZYtW1KjRg3Wrl0LwP33329Jwpg0CmqiEJHmIrJFRLaLyJN+5msnIioi1n+USbMzZ84wfPhwKlWqxJIlSxg5ciQVK8XkI2kAABxNSURBVFb0OixjMrxA+noCQERyq+rpi5g/OzAeaALEAD+LyFzffqPc+QoCfYGfAl23MUnFx8dTr149Vq5cSdu2bRkzZgylSpXyOixjMoVAuhmvLSLrgG3ucDUR+U8A666N8+yKHap6BpgOJNez2lBgBHAq8LCNcRw5cgSA7Nmz8+CDD/L555/z6aefWpIw5hIKpOppLNASOAigqmtwnniXmhLALp/hGJI8a1tEagClVPUL/BCRHiISLSLR9lxiA05j9dSpUylTpgxz5swBoHfv3rRs2dLjyIzJfAJJFNlU9Y8k4+IDWC656w8Tu6oVkWw4z7oYmNqKVHWSqkapalTOnPa47qxu48aNNGrUiK5duxIREUHZsmW9DsmYTC2QRLFLRGoDKiLZRaQ/sDWA5WIA3/P/ksAen+GCQGVgsYj8DtQB5lqDtvFnxIgRVKtWjfXr1zN58mSWLl1K5cqVvQ7LmEwtkETRCxgAXAP8hXNA7xXAcj8D5UWktPugo3uBuQkTVTVWVcNUNVxVw4HlQCtVjb7Iz2CygITnplx11VV07NiRzZs3061bN+vAz5h0kOqDi/7VykVaAGOA7MAUVR0mIi8B0ao6N8m8i4HHU0sU9uCirGXPnj3069ePG2+8kb59+3odjjEZ1r95cFGql8eKyP/h07aQQFV7pLasqn6J0+us77jnUpi3UWrrM1lHQgd+Q4YMIS4ujnr16nkdkjFZViD3USz0eZ8HuJPzr2Yy5pJavXo13bt3Z+XKlTRt2pQJEyZYg7UxHko1UajqDN9hEXkfWBC0iEyWFxsby549e5gxYwZ33323deBnjMcCvjPbR2ng2ksdiMm6VJWZM2eybds2hgwZwk033cSOHTvIkyeP16EZYwjszuxDIvK3+zqMczbxdPBDM1nBr7/+SosWLWjfvj1z5swh4YZKSxLGhA6/iUKcc/5qwOXuq4iqllHVj9MjOJN5nT59mmHDhlG5cmV++OEH3nzzTX788UfshkpjQo/fqidVVRGZpaq10isgkzXs2rWLoUOHcscddzBmzBhKlCiR+kLGGE8EcrfSChGpGfRITKa3f/9+xo1zHpRYrlw5Nm7cyMyZMy1JGBPiUkwUIpJwttEAJ1lsEZFfRGSViPySPuGZzODcuXO8/fbbREREMGDAALZs2QJAmTJlPI7MGBMIf1VPK4CaQJt0isVkQuvXr6dXr158//333HjjjUycOJEKFSp4HZYx5iL4SxQCoKq/plMsJpM5c+YMTZs25cyZM0yZMoUHHnjA7okwJgPylyguF5EBKU1U1VFBiMdkAt999x033XQTuXLl4uOPPyYiIoKwsDCvwzLGpJG/xuzsQAGc7sCTexlznpiYGO666y4aN27Me++9B0CDBg0sSRiTwfk7o9irqi+lWyQmwzp79izjxo3j2WefJT4+nldffZWOHTt6HZYx5hJJtY3CmNTcf//9TJ8+ndtuu43x48dTunRpr0MyxlxCKT6PQkSKqurf6RxPqux5FKHh8OHD5MiRgwIFCvD999/z559/ctddd1ljtTEh6t88jyLFNopQTBLGe6rK9OnTiYyM5NlnnwWcdoh27dpZkjAmk7LnSJqAbd++nWbNmtGhQwdKlixJp06dvA7JGJMOLFGYgHz44YdUrlyZn376iXHjxrF8+XJq1bIuwIzJCtLyPAqThcTFxZEzZ06ioqJo164dI0aM4Oqrr/Y6LGNMOkqxMTtUWWN2+ti3bx8DBw7k+PHjfPbZZ16HY4z5l4LSmG2ypnPnzjFp0iQqVKjAjBkzqFSpEvHx8V6HZYzxkFU9mUQ7duygU6dOLFu2jEaNGvHf//6XiIgIr8MyxnjMEoVJVLhwYQ4fPsy7777L/fffb5e7GmMAq3rK8ubOnUvbtm2Jj4+nWLFirF+/ns6dO1uSMMYkskSRRe3cuZM2bdrQunVrtm7dyt69ewHIls12CWPM+eyokMWcPXuWkSNHEhkZyfz583nttddYtWoVJUuW9Do0Y0yIsjaKLCY+Pp7Jkydzyy238J///Ifw8HCvQzLGhDg7o8gCDh06xODBgzl69Ci5c+fmhx9+YO7cuZYkjDEBsUSRiakqH3zwAREREbzxxhssWrQIgGLFilljtTEmYJYoMqmtW7fSpEkTOnXqRHh4ONHR0bRq1crrsIwxGZC1UWRS/fv3Jzo6mgkTJtCjRw+yZ8/udUjGmAzKEkUmsmDBAiIiIihVqhT//e9/yZ07N1dddZXXYRljMrigVj2JSHMR2SIi20XkyWSmDxCRjSKyVkS+FZFrgxlPZvXnn39y33330bRpU1577TUArr32WksSxphLImiJQkSyA+OB24CKQAcRqZhktlVAlKpWBT4BRgQrnszo3LlzTJw4kYiICD799FOef/55Ro4c6XVYxphMJphnFLWB7aq6Q1XPANOB1r4zqOoiVT3hDi4H7K6vi/Dqq6/Sq1cvatWqxdq1a3nhhRfIkyeP12EZYzKZYLZRlAB2+QzHADf4mb8b8FVyE0SkB9ADoEDxspcqvgzp6NGjHDhwgNKlS9OzZ09Kly5Nhw4d7HJXY0zQBPOMIrkjV7JPSRKRTkAU8Hpy01V1kqpGqWpUzpw5L2GIGYeqMmvWLCpWrEj79u1RVYoVK8Z9991nScIYE1TBTBQxQCmf4ZLAnqQzicitwBCglaqeDmI8GdYff/xBq1ataNu2LUWLFmXs2LGWHIwx6SaYVU8/A+VFpDSwG7gXuM93BhGpAbwFNFfVfUGMJcNatmwZt956KwAjR46kX79+5MhhVzUbY9JP0M4oVPUs8CjwDbAJ+FhVN4jISyKScIvw60ABYKaIrBaRucGKJ6M5cuQIADVr1uTBBx9k06ZNDBw40JKEMSbdiWqyzQYhq+i1kfr3H5u8DiNoDh48yJNPPsn8+fPZsGEDBQoU8DokY0wmICIrVTUqLctaX08hQlV57733iIiI4J133qF9+/bWDmGMCQlWjxECYmNjadOmDYsXL6Zu3bpMnDiRqlWreh2WMcYAlig8paqICIUKFSIsLIxJkybRrVs3exypMSak2BHJI9988w01a9YkJiYGEWHmzJk89NBDliSMMSHHjkrpbO/evdx77700b96cEydOsG+fXRVsjAltlijS0fjx44mIiGD27Nm8+OKLrF27lpo1a3odljHG+GVtFOlo5cqV3HDDDYwfP57y5ct7HY4xxgTEziiC6MiRI/Tv35+VK1cCMGHCBL755htLEsaYDMUSRRCoKp988gmRkZGMHTuWJUuWAJAnTx67N8IYk+FYorjEfvvtN1q2bMndd9/NFVdcwbJlyxgwYIDXYRljTJpZorjEPvjgA5YuXcro0aP5+eefueEGf4/gMMaY0Gd9PV0C//vf/zh9+jS33norp0+fZv/+/ZQsaQ/rM8aEDuvrySMHDhzgwQcfpGHDhrz00ksA5M6d25KEMSZTsctj00BVmTp1KoMGDSI2NpbBgwfz7LPPeh2WCTFxcXHExMRw6tQpr0MxWUiePHkoWbIkl/JpoJYo0uDLL7/kwQcfpH79+kycOJHKlSt7HZIJQTExMRQsWJDw8HC72s2kC1Xl4MGDxMTEULp06Uu2Xqt6CtCJEyf44YcfAGjRogVz5sxh6dKlliRMik6dOkWxYsUsSZh0IyIUK1bskp/FWqIIwFdffUXlypW57bbbOHz4MCJCq1atrAM/kypLEia9BWOfsyOdH7t37+buu++mRYsW5M6dm88//5zLLrvM67CMMSZdWaJIwb59+6hYsSJffPEFL7/8MmvWrOGmm27yOixjLkr27NmpXr06lStX5o477uDw4cOJ0zZs2MAtt9zCddddR/ny5Rk6dCi+l8t/9dVXREVFERkZSUREBI8//rgXH8GvVatW0b17d6/DSNHBgwe5+eabKVCgAI8++miK8/399980adKE8uXL06RJEw4dOgQ4bQ59+/alXLlyVK1alV9++QWA/fv307x583T5DGCJ4gK7d+8G4IorrmDo0KGsX7+eIUOGkCtXLo8jM+bi5c2bl9WrV7N+/XqKFi3K+PHjATh58iStWrXiySefZOvWraxZs4Yff/yRCRMmALB+/XoeffRRpk2bxqZNm1i/fj1lypS5pLGdPXv2X6/jlVdeoU+fPum6zYuRJ08ehg4dysiRI/3ON3z4cBo3bsy2bdto3Lgxw4cPB5xkvW3bNrZt28akSZPo1asXAJdffjnFixdPbDcNNrvqyRUbG8szzzzDW2+9xfLly6lZsyZ9+/b1OiyTSbz4+QY27jlySddZ8epCPH9HpYDnr1u3LmvXrgXgww8/pH79+jRt2hSAfPnyMW7cOBo1asQjjzzCiBEjGDJkCBEREQDkyJGD3r17X7DOY8eO0adPH6KjoxERnn/+ee666y4KFCjAsWPHAPjkk0/44osvmDp1Kg888ABFixZl1apVVK9enVmzZrF69erEKt1y5crxww8/kC1bNnr27MnOnTsBGDNmDPXr1z9v20ePHmXt2rVUq1YNgBUrVtC/f39OnjxJ3rx5eeedd6hQoQJTp05l3rx5nDp1iuPHj/Pdd9/x+uuv8/HHH3P69GnuvPNOXnzxRQDatGnDrl27OHXqFP369aNHjx4Bl29y8ufPT4MGDdi+fbvf+ebMmcPixYsB6NKlC40aNeK1115jzpw5dO7cGRGhTp06HD58mL1791K8eHHatGnDBx98cEG5BEOWTxSqysyZM+nfvz9//vknjz76KGXLlvU6LGMuqfj4eL799lu6desGONVOtWrVOm+esmXLcuzYMY4cOcL69esZOHBgqusdOnQohQsXZt26dQCJVSb+bN26lYULF5I9e3bOnTvHrFmz6Nq1Kz/99BPh4eFceeWV3HfffTz22GM0aNCAnTt30qxZMzZtOr9Hhujo6POuOoyIiGDp0qXkyJGDhQsX8vTTT/Ppp58CsGzZMtauXUvRokWZP38+27ZtY8WKFagqrVq1YunSpTRs2JApU6ZQtGhRTp48yfXXX89dd91FsWLFztvuY489xqJFiy74XPfeey9PPvlkqp8/OX/99RfFixcHoHjx4okPNNu9ezelSpVKnK9kyZLs3r2b4sWLExUVxTPPPJOm7V2sLJ0oVJW2bdsye/Zsatasydy5c4mKStMd7sb4dTG//C+lkydPUr16dX7//Xdq1apFkyZNgH+e156ci7lqZuHChUyfPj1xuEiRIqkuc/fdd5M9e3YA2rdvz0svvUTXrl2ZPn067du3T1zvxo0bE5c5cuQIR48epWDBgonj9u7dy+WXX544HBsbS5cuXdi2bRsiQlxcXOK0Jk2aULRoUQDmz5/P/PnzqVGjBuCcFW3bto2GDRsyduxYZs2aBcCuXbvYtm3bBYli9OjRgRXOJZBcF0sJ/58rrriCPXv2pEscWTJRxMXFkTNnTkSEBg0acMstt9C7d+/EndeYzCKhjSI2NpaWLVsyfvx4+vbtS6VKlVi6dOl58+7YsYMCBQpQsGBBKlWqxMqVKxOrdVKSUsLxHZf0mv78+fMnvq9bty7bt29n//79zJ49O/EX8rlz51i2bBl58+b1+9l81/3ss89y8803M2vWLH7//XcaNWqU7DZVlaeeeoqHH374vPUtXryYhQsXsmzZMvLly0ejRo2SvR8hGGcUV155ZWKV0t69e7niiisA5wxi165difPFxMRw9dVXA065+iufSynLNWYvXryYqlWrMmfOHAAGDhxInz59LEmYTK1w4cKMHTuWkSNHEhcXR8eOHfn+++9ZuHAh4Jx59O3blyeeeAKAQYMG8corr7B161bAOXCPGjXqgvU2bdqUcePGJQ4nVD1deeWVbNq0KbFqKSUiwp133smAAQOIjIxM/PWedL2rV6++YNnIyMjz6v5jY2MpUaIEAFOnTk1xm82aNWPKlCmJbSi7d+9m3759xMbGUqRIEfLly8fmzZtZvnx5ssuPHj2a1atXX/BKa5IAaNWqFe+++y4A7777Lq1bt04c/95776GqLF++nMKFCydWUW3dujXdbvjNMoli//79dOnShZtvvpnTp0+fdwprTFZQo0YNqlWrxvTp08mbNy9z5szh5ZdfpkKFClSpUoXrr78+8RLOqlWrMmbMGDp06EBkZCSVK1dm7969F6zzmWee4dChQ1SuXJlq1aol/tIePnw4LVu25JZbbkk8sKWkffv2TJs2LbHaCWDs2LFER0dTtWpVKlasyMSJEy9YLiIigtjYWI4ePQrAE088wVNPPUX9+vWJj49PcXtNmzblvvvuo27dulSpUoV27dpx9OhRmjdvztmzZ6latSrPPvssderUSb1QAxAeHs6AAQOYOnUqJUuWTKxS6969O9HR0QA8+eSTLFiwgPLly7NgwYLEpNOiRQvKlClDuXLleOihhxKvSgNYtGgRt99++yWJMTVZopvxjz76iEceeYRjx44xaNAghgwZQr58+YIUoTGOTZs2ERkZ6XUYmdro0aMpWLBgSN9LESwNGzZkzpw5ybYLJbfvWTfjqTh79iyVK1dm9erVDBs2zJKEMZlEr169yJ07t9dhpLv9+/czYMCAgC4euBQy5RnF8ePHGTp0KNdccw29e/dOvHLA+t0x6cnOKIxX7IwiFV988QWVKlXitddeS2yIExFLEsYTGe2HmMn4grHPZZpEERMTQ9u2bbnjjjvInz8/S5cuZcyYMV6HZbKwPHnycPDgQUsWJt0kPI8iT548l3S9meY+ih07dvDNN9/w6quvMmDAAOubyXiuZMmSxMTEsH//fq9DMVlIwhPuLqUM3UaxYsUKli1bRr9+/QCnp8akd1EaY4wJ4TYKEWkuIltEZLuIXHA3iojkFpEZ7vSfRCQ8kPUePnyY3r17U6dOHUaNGsXx48cBLEkYY0wQBC1RiEh2YDxwG1AR6CAiFZPM1g04pKrlgNHAa6mt98yJWCIiInjrrbfo27cv69atO+/2fGOMMZdWMM8oagPbVXWHqp4BpgOtk8zTGnjXff8J0FhSuTzp+IE/KVWqFD///DNjxoyhUKFClzxwY4wx/whmY3YJYJfPcAxwQ0rzqOpZEYkFigEHfGcSkR5AQsfwp6Ojo9cn7SI5iwojSVllYVYW/7Cy+IeVxT8qpHXBYCaK5M4MkracBzIPqjoJmAQgItFpbZDJbKws/mFl8Q8ri39YWfxDRKLTumwwq55igFI+wyWBpJ2nJ84jIjmAwsDfQYzJGGPMRQpmovgZKC8ipUUkF3AvMDfJPHOBLu77dsB3mtGu1zXGmEwuaFVPbpvDo8A3QHZgiqpuEJGXgGhVnQu8DbwvIttxziTuDWDVk4IVcwZkZfEPK4t/WFn8w8riH2kuiwx3w50xxpj0lWn6ejLGGBMcliiMMcb4FbKJIljdf2REAZTFABHZKCJrReRbEbnWizjTQ2pl4TNfOxFREcm0l0YGUhYico+7b2wQkQ/TO8b0EsB35BoRWSQiq9zvSQsv4gw2EZkiIvtEZH0K00VExrrltFZEaga0YlUNuRdO4/evQBkgF7AGqJhknt7ARPf9vcAMr+P2sCxuBvK573tl5bJw5ysILAWWA1Fex+3hflEeWAUUcYev8DpuD8tiEtDLfV8R+N3ruINUFg2BmsD6FKa3AL7CuYetDvBTIOsN1TOKoHT/kUGlWhaqukhVT7iDy3HuWcmMAtkvAIYCI4BT6RlcOgukLB4CxqvqIQBV3ZfOMaaXQMpCgYT+fgpz4T1dmYKqLsX/vWitgffUsRy4TESKp7beUE0UyXX/USKleVT1LJDQ/UdmE0hZ+OqG84shM0q1LESkBlBKVb9Iz8A8EMh+cR1wnYj8ICLLRaR5ukWXvgIpixeATiISA3wJ9Emf0ELOxR5PgNB9cNEl6/4jEwj4c4pIJyAKuCmoEXnHb1mISDacXogfSK+APBTIfpEDp/qpEc5Z5v9EpLKqHg5ybOktkLLoAExV1TdEpC7O/VuVVfVc8MMLKWk6bobqGYV1//GPQMoCEbkVGAK0UtXT6RRbekutLAoClYHFIvI7Th3s3EzaoB3od2SOqsap6m/AFpzEkdkEUhbdgI8BVHUZkAenw8CsJqDjSVKhmiis+49/pFoWbnXLWzhJIrPWQ0MqZaGqsaoapqrhqhqO017TSlXT3BlaCAvkOzIb50IHRCQMpypqR7pGmT4CKYudQGMAEYnESRRZ8Rm1c4HO7tVPdYBYVd2b2kIhWfWkwev+I8MJsCxeBwoAM932/J2q2sqzoIMkwLLIEgIsi2+ApiKyEYgHBqnqQe+iDo4Ay2Ig8H8i8hhOVcsDmfGHpYh8hFPVGOa2xzwP5ARQ1Yk47TMtgO3ACaBrQOvNhGVljDHmEgrVqidjjDEhwhKFMcYYvyxRGGOM8csShTHGGL8sURhjjPHLEoUJOSISLyKrfV7hfuYNT6mnzIvc5mK399E1bpcXFdKwjp4i0tl9/4CIXO0zbbKIVLzEcf4sItUDWKa/iOT7t9s2WZclChOKTqpqdZ/X7+m03Y6qWg2ns8nXL3ZhVZ2oqu+5gw8AV/tM666qGy9JlP/EOYHA4uwPWKIwaWaJwmQI7pnD/0TkF/dVL5l5KonICvcsZK2IlHfHd/IZ/5aIZE9lc0uBcu6yjd1nGKxz+/rP7Y4fLv88A2SkO+4FEXlcRNrh9Ln1gbvNvO6ZQJSI9BKRET4xPyAi/0ljnMvw6dBNRP4rItHiPHviRXdcX5yEtUhEFrnjmorIMrccZ4pIgVS2Y7I4SxQmFOX1qXaa5Y7bBzRR1ZpAe2BsMsv1BN5U1eo4B+oYt7uG9kB9d3w80DGV7d8BrBORPMBUoL2qVsHpyaCXiBQF7gQqqWpV4GXfhVX1EyAa55d/dVU96TP5E6Ctz3B7YEYa42yO001HgiGqGgVUBW4SkaqqOhanL5+bVfVmtyuPZ4Bb3bKMBgaksh2TxYVkFx4myzvpHix95QTGuXXy8Tj9FiW1DBgiIiWBz1R1m4g0BmoBP7vdm+TFSTrJ+UBETgK/43RDXQH4TVW3utPfBR4BxuE862KyiMwDAu7SXFX3i8gOt5+dbe42fnDXezFx5sfprsL3CWX3iEgPnO91cZwH9KxNsmwdd/wP7nZy4ZSbMSmyRGEyiseAv4BqOGfCFzyUSFU/FJGfgNuBb0SkO063yu+q6lMBbKOjbweCIpLs803cvoVq43Qydy/wKHDLRXyWGcA9wGZglqqqOEftgOPEeYrbcGA80FZESgOPA9er6iERmYrT8V1SAixQ1Q4XEa/J4qzqyWQUhYG97vMD7sf5NX0eESkD7HCrW+biVMF8C7QTkSvceYpK4M8U3wyEi0g5d/h+YIlbp19YVb/EaShO7sqjozjdnifnM6ANzjMSZrjjLipOVY3DqUKq41ZbFQKOA7EiciVwWwqxLAfqJ3wmEcknIsmdnRmTyBKFySgmAF1EZDlOtdPxZOZpD6wXkdVABM4jHzfiHFDni8haYAFOtUyqVPUUTu+aM0VkHXAOmIhz0P3CXd8SnLOdpKYCExMas5Os9xCwEbhWVVe44y46Trft4w3gcVVdg/N87A3AFJzqrASTgK9EZJGq7se5IusjdzvLccrKmBRZ77HGGGP8sjMKY4wxflmiMMYY45clCmOMMX5ZojDGGOOXJQpjjDF+WaIwxhjjlyUKY4wxfv0/LegVW2wodLgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred = model.predict(x_test[0:100])\n",
    "pred = pred[:,1] # Only positive class (M)\n",
    "plot_roc(pred,y_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[43  1]\n",
      " [ 0 56]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAEYCAYAAAAaryJBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAf00lEQVR4nO3dd5xkRb3+8c+zuyAgOYqggooYUBAWLoIoQblgAgNBkQuIclExgQERSdesPzOGRZQ1wwUxgArIBRREwi5ZVARFCZJBQEB2eX5/nBrpHWd6ume7+5zped68+jV9Up3qpV/znW9VnSrZJiIiYtBm1F2BiIiYnhKAIiKiFglAERFRiwSgiIioRQJQRETUYlbdFYiIiP6ZufyT7AUPdHWNH7jtNNs79KlK/5IAFBExxLzgAR6z/q5dXfPgpUev2qfqLCIBKCJiqAnUzN6WBKCIiGEmQKq7FmNKAIqIGHbJgCIiohbJgCIiYvDSBxQREXVJBhQREQMnkgFFREQdlAwoIiJqkgwoIiJqkQwoIiIGL6PgIiKiDpkJISIiapMMKCIiBi9NcBERUZcZaYKLiIhBy4OoERFRmwxCiIiIwUsfUERE1CUZUERE1CIZUEREDJwyGWlERNQlGVBERNQiGVBERAxef0bBSfozcC+wEFhge7aklYHjgXWAPwO72r5rvDKamZdFRETvjPQDdfrq3Da2N7I9u2wfDJxpez3gzLI9rgSgiIhhNjITQjevydsJmFvezwV2bndyAlBExFDTZALQqpIubnntN0bBBk6XNK/l+Bq2bwYoP1dvV7P0AUVEDLvuByHc3tKsNp4tbd8kaXXgDEm/6/YmCUAREcOuD4MQbN9Uft4q6WRgM+AWSWvavlnSmsCt7cpIE1xExLDr8SAESY+VtNzIe2B74Ergx8Be5bS9gB+1KycZUETEMFNfhmGvAZysKljNAr5r++eSLgJOkLQv8Bdgl3aFJABFRAy7Hj+Iavs6YMMx9t8BbNdpOQlAERFDTpkJISIiBk0kAEVERB1UXg2UUXAREVGLZEAREUNNaYKLiIh6JABFREQtEoAiIqIWCUARETF4GQUXUS9JS0v6iaR7JP3vYpSzh6TTe1m3ukjaStLv665H9JfKIIRuXoOSABSNIul1Zf2R+yTdLOlnkp7fg6JfQzV/1Sq2285P1Y7t79jevgf16StJlvTUdufY/pXt9QdVp6hPAlDEBCQdCHwW+AhVsHgi8CWqVRYX15OAP9he0IOypjxJaX6fRhKAItqQtAJwFPBW2z+wfb/th23/xPZ7yjmPkfRZSTeV12clPaYc21rSDZIOknRryZ72KceOBA4DdiuZ1b6SjpD07Zb7r1Oyhllle29J10m6V9KfJO3Rsv/cluu2kHRRadq7SNIWLcfOlvQ/ks4r5ZwuadVxPv9I/d/bUv+dJb1E0h8k3SnpkJbzN5N0vqS7y7lflLRkOfbLctpl5fPu1lL++yT9DfjGyL5yzVPKPTYu24+XdLukrRfrf2w0QgJQRHvPA5YCTm5zzgeAzYGNqGbi3Qw4tOX444AVgLWAfYGjJa1k+3CqrOp428vaPrZdRVStb/J5YEfbywFbAJeOcd7KwKnl3FWATwOnSlql5bTXAftQLU28JPDuNrd+HNW/wVpUAfMY4PXAJsBWwGGSnlzOXQi8C1iV6t9uO+AtALZfUM7ZsHze41vKX5kqG1xkiWXb1wLvA74jaRngG8Bxts9uU9+YCjSJ14AkAEVTrEK1DHC7JrI9gKNs32r7NuBIYM+W4w+X4w/b/ilwHzDZPo5HgA0kLW37ZttXjXHOS4FrbH/L9gLb3wN+B7y85Zxv2P6D7QeAE6iC53geBj5s+2Hg+1TB5XO27y33vwp4DoDtebZ/U+77Z+CrwAs7+EyH236o1GcRto8BrgEuANakCvgxBJIBRbR3B7DqBH0Tjweub9m+vuz7VxmjAtg/gGW7rYjt+4HdgP2BmyWdKunpHdRnpE5rtWz/rYv63GF7YXk/EiBuaTn+wMj1kp4m6RRJf5P0d6oMb8zmvRa32X5wgnOOATYAvmD7oQnOjSkgo+AiJnY+8CCwc5tzbqJqPhrxxLJvMu4HlmnZflzrQdun2X4xVSbwO6pfzBPVZ6RON06yTt34MlW91rO9PHAIEzeeuN1BSctSDQI5FjiiNDHGEEgAimjD9j1U/R5Hl873ZSQtIWlHSZ8op30POFTSaqUz/zDg2+OVOYFLgRdIeqKqARDvHzkgaQ1Jryh9QQ9RNeUtHKOMnwJPUzV0fJak3YBnAqdMsk7dWA74O3Bfyc7ePOr4LcCT/+2q9j4HzLP9Rqq+ra8sdi2jGdIHFNGe7U8DB1INLLgN+CtwAPDDcsqHgIuBy4ErgPll32TudQZwfClrHosGjRnAQVQZzp1UfStvGaOMO4CXlXPvAN4LvMz27ZOpU5feTTXA4V6q7Oz4UcePAOaWUXK7TlSYpJ2AHaiaHaH6/7Cxyui/mMLU3AxIdtusPCIiprAlVnuKV9n5411dc8vXdplne3afqvQveRgtImLIDTKr6ca0aoJT9WBg36N6RERTNHkU3JTJgCTNyjQqERGT0MwEaLABSNI6wM+Ac6meLr+Rap6v9alG3CwDXAu8wfZdks4Gfg1sCfxY0rOpnoV4OtXw132AvaieBL/A9t7lPl8GNgWWBk4sT8JHREw/am4TXB0Z0HrAa22/SdIJwKupRg+9zfY5ko4CDgfeWc5f0fYLASQdB6wEbAu8AvgJVXB6I3CRpI1sXwp8wPadkmYCZ0p6ju3Lx6uQpP0oU5No1lKbzFpprfFOjZjQBk9Yse4qxBC4ZP68222v1ouyEoAe9acSJKAa/voUqiBzTtk3F2hdr2X08NKf2LakK4BbbF8BIOkqYB2q5zt2LUFlFtWDhM+kGm47JttzgDkAS67+VD9ut08vxseL6e6Xn3pZ3VWIIbDcUjNHz7IxaQlAj2qd3mMhMNGfi/ePc/0jo8p6BJglaV2qZyQ2Lc14x1FN8BgRMT01M/40YhTcPcBdkrYq23sC57Q5fyLLUwWteyStAey4mPWLiJjSMgquvb2Ar6iaBv46qsEFk2L7MkmXUM0cfB1wXm+qGBEx9Qw6qHRjoAGoTBu/Qcv2p1oObz7G+VuP2t67TVl7j/W+XXkREdNBAlBERNQiASgiIurRzPiTABQRMeySAUVExOBlJoSIiKiDgIbGnwSgiIjhlmHYERFRk4bGnwSgiIhh19QMqAlT8URERL+oyoC6eXVctDRT0iWSTinb60q6QNI1ko6XtGS76xOAIiKGmIAZM9TVqwvvAK5u2f448Bnb6wF3Afu2uzgBKCJiyPUjA5K0NvBS4GtlW1RrtZ1YTpkL7NyujPQBRUQMuUn0Aa0q6eKW7Tll3bRWn6VaTHS5sr0KcLftBWX7BqDt6p4JQBERw6zLfp3idtuzxy1Sehlwq+15krZ+9E7/xu1ukgAUETHEqgdRez4KbkvgFZJeQrXg5/JUGdGKkmaVLGht4KZ2haQPKCJiqHW3GF0nwcr2+22vbXsdYHfg/2zvAZwFvKacthfwo3blJABFRAy5fg3DHsP7gAMl/ZGqT+jYdienCS4iYsj180FU22cDZ5f31wGbdXptAlBExDBb/KymbxKAIiKGWJ8GIfREAlBExJBraPxJAIqIGHbJgCIiohYNjT8JQBERQy1LckdERB2yJHdERNQkS3JHRERNGhp/EoAiIoZdMqCIiBi8zIQQERF1yEwIERFRmwSgiIioRUPjTwJQRMSwa2oGlAXpIiKiFsmAIiKGWUbBRUREHZSZECIioi4NjT8JQBERw25GQyNQAlBExJBraPxJAIqIGGbKekAREVGXGc2MPwlAERHDLhlQRETUoqHxJwEoImKYiepZoCYaNwBJWr7dhbb/3vvqREREr03FPqCrAMMioXNk28AT+1iviIjoBU3BmRBsP2GQFYmIiP5oaPzpbDZsSbtLOqS8X1vSJv2tVkRE9IKoZkLo5jUoEwYgSV8EtgH2LLv+AXyln5WKiIjekbp7DUono+C2sL2xpEsAbN8pack+1ysiInpkyvUBtXhY0gyqgQdIWgV4pK+1ioiInhh0VtONTgLQ0cBJwGqSjgR2BY7sa60iIqJnpuxs2La/KWke8KKyaxfbV/a3WhER0SvNDD+dz4QwE3iYqhmuo5FzERHRDE3tA+pkFNwHgO8BjwfWBr4r6f39rlhERCy+ahh2d69B6SQDej2wie1/AEj6MDAP+Gg/KxYRET3Qh5kQJC0F/BJ4DFUcOdH24ZLWBb4PrAzMB/a0/c/xyumkOe16Fg1Us4DrJlvxiIgYrD48B/QQsK3tDYGNgB0kbQ58HPiM7fWAu4B92xXSbjLSz1D1+fwDuErSaWV7e+DcjqoYERG163UGZNvAfWVzifIysC3wurJ/LnAE8OXxymnXBDcy0u0q4NSW/b/pvroREVGHkT6gLq0q6eKW7Tm25yxSrjSTqjvmqVSP61wL3G17QTnlBmCtdjdpNxnpsV1XOSIiGmcSGdDttme3O8H2QmAjSSsCJwPPGOu0dmVMOAhB0lOADwPPBJZqufnTJro2IiLq18+BbbbvlnQ2sDmwoqRZJQtaG7ip3bWdDEI4DvgG1WfYETiBapRDREQ0nNT72bAlrVYyHyQtTTVRwdXAWcBryml7AT9qV04nAWgZ26cB2L7W9qFUs2NHRMQU0IdRcGsCZ0m6HLgIOMP2KcD7gAMl/RFYBWjbldPJc0APqWpAvFbS/sCNwOodVTEiImrXh1FwlwPPHWP/dcBmnZbTSQB6F7As8HaqvqAVgDd0eoOIiKhXQ2fi6Wgy0gvK23t5dFG6iIiYAsRgVzntRrsHUU+mzRA626/qS40iIqJ3puh6QF8cWC0a5NlPWJHzPvOKuqsRU9hKmx5QdxUiFtHU2bDbPYh65iArEhER/dHUNXQ6XQ8oIiKmIDEFM6CIiBgOg1zjpxsdByBJj7H9UD8rExERvdfUANTJiqibSboCuKZsbyjpC32vWURELLZqdgN19RqUTvqmPg+8DLgDwPZlZCqeiIgpYyovyT3D9vWjouLCPtUnIiJ6rKFjEDoKQH+VtBngsgDR24A/9LdaERHRC9WCdM2MQJ0EoDdTNcM9EbgF+EXZFxERU8CUfQ7I9q3A7gOoS0RE9EFDE6COVkQ9hjHmhLO9X19qFBERPaMOF5mrQydNcL9oeb8U8Ergr/2pTkRE9FpD409HTXDHt25L+hZwRt9qFBERPdXUB1EnMxXPusCTel2RiIjovSk9Ck7SXTzaBzQDuBM4uJ+VioiI3mlo/GkfgFQ9fbohcGPZ9YjtcRepi4iIhhnw7AbdaDs8vASbk20vLK8En4iIKUZd/jconTyfdKGkjftek4iI6LmqD2iKzQUnaZbtBcDzgTdJuha4n+rz2HaCUkTEFNDUJrh2fUAXAhsDOw+oLhER0QdTcUVUAdi+dkB1iYiIHhtpgmuidgFoNUkHjnfQ9qf7UJ+IiOglTc1h2DOBZWGAQyIiIqLnpuKDqDfbPmpgNYmIiJ5rchNcu2HYDa1yREQMg3YZ0HYDq0VERPRNQ1vgxg9Atu8cZEUiIqIfxIyGNmhNZjbsiIiYIsQUzIAiImIINHgy0gSgiIghNxWHYUdExBSXJriIiKhNMqCIiKhFQ+NPAlBExDATnS38Voem1isiInpB1XIM3bwmLFJ6gqSzJF0t6SpJ7yj7V5Z0hqRrys+V2pWTABQRMeTU5asDC4CDbD8D2Bx4q6RnAgcDZ9peDzizbI8rASgiYohVk5Gqq9dEbN9se355fy9wNbAWsBMwt5w2lwkWNE0fUETEkJvEGIRVJV3csj3H9pwxy5bWAZ4LXACsYftmqIKUpNXb3SQBKCJiyE1iFNzttmdPXK6WBU4C3mn7790u/Z0AFBEx1DobWNB1qdISVMHnO7Z/UHbfImnNkv2sCdzaroz0AUVEDLGRYdjdvCYss4poxwJX2/50y6EfA3uV93sBP2pXTjKgiIgh14cMaEtgT+AKSZeWfYcAHwNOkLQv8Bdgl3aFJABFRAy5Xocf2+e2KbbjxUwTgCIihpn6kgH1RAJQRMQQa/JUPAlAERFDLhlQRETUopnhJwEoImLoNTQBSgCKiBhmVR9QMyNQAlBExJBLBhQRETUQSgYUERF1SAYUEREDlz6giIioh5IBRURETRKAIiKiFhmEEBERAydgRjPjTwJQRMSwSwYUERG1SB9QRETUIhlQREQMXPqAIiKiJpmKJyIi6pAHUSMioi4NjT8JQBERw6zqA2pmCEoAiogYcs0MPwlAERHDr6ERKAEoImLIZRRcRETUoqFdQMzoV8GSft3BOe+UtEwP7vVnSasubjkREcNIXb4GpW8ByPYWHZz2TmDMACRpZm9rNJiyIyIap6ERqJ8Z0H3l59aSzpZ0oqTfSfqOKm8HHg+cJemskWskHSXpAuB5rZmNpNmSzi7vV5F0uqRLJH2Vln8ySa+XdKGkSyV9dSTYjC67X587IqJJqpjS3X+DMqg+oOcCzwJuAs4DtrT9eUkHAtvYvr2c91jgStuHAWj8hsvDgXNtHyXppcB+5fxnALuV8h+W9CVgD+Cbo8tuJWm/kTKA+5ZeQr9f7E883FYFbp/wrIjx5Ts0sSf1pJTMhMCFtm8AkHQpsA5w7hjnLQRO6qC8FwCvArB9qqS7yv7tgE2Ai0rwWhq4daKybc8B5nTyQQIkXWx7dt31iKkr36HBamj8GVgAeqjl/cI2933Q9sKW7QU82ky41KhzPcb1Aubafn8HZUdETA8NjUB96wPq0L3Acm2O/5kqowF4dcv+X1I1rSFpR2Clsv9M4DWSVi/HVpbUmzQ2ImJK6rYHaHDRqu4ANAf42cgghDEcCXxO0q+oMqfW/S+QNB/YHvgLgO3fAocCp0u6HDgDWLNflZ/G0lwZiyvfoQGSunsNrF72WC1ZERExDJ71nI393VPO6eqajZ60/LxB9NFlJoSIiGHX0D6gBKCIiCHX1Lng6u4DioiIPut1H5Ckr0u6VdKVLftWlnSGpGvKz5XalQEJQNEHkvK9isWmlifR1eap9JhYH2biOQ7YYdS+g4Ezba9HNSL54IkKyS+K6ClJmwL79GKS2Zi+JM1wGSEl6dVUM6nEZHQbfTqIQLZ/Cdw5avdOwNzyfi6w80TlJABFry0L7A/sKmnpuisTU5PtRwAkbQ7sCdxYb42mnVUlXdzy2m/iS1jD9s0A5efqE12QQQjRU7bPkvQeqvn6Zkr6ru0H6q5XTC2lyW1r4HTgANt3SVrK9oP11mxqmsQghNsHMQw7GVAsttHt87bPBo6g+sv1dcmEohOt3yNXzgK+DXyw7HswS6l0TwzsQdRbJK0JUH7eOsH5CUCxeCSppa1+F0kHSZpt+xzg/VRBaPf0CUU7o75Hr5S0d/ke7QP8X1leZabthQlC3RvQckA/BvYq7/cCfjTRBQlAsVhafmkcQLXA4CPAtyS9BbiAaiTM21l0Lr+IRbR8jw6i+r48Hjha0n/a/i/gUuBPZXBCJhXuVo8jkKTvAecD60u6QdK+wMeAF0u6Bnhx2W4rfUCx2CRtDGxDtRzGvsA/ga2AWWXdpzcBt9VYxWgoSSvbvrM0vz0OeK7tbUo/4h3A2QC295b0FaqlXK6rq75TVa8fRLX92nEObddNOcmAomtj9PnMB94KvBB4pe0NqdZ7ep+kfW1fbPv6GqoaDVVWRV4LmCdp+5IBPViOnUD1XdrJ9kOS9pS0lu39bSf4TEJTJyNNAIqutTSX7ChppzI66W/AysDd5bQbqZbNOKWmakazyfaNwCHA/5P0Att3AZdRZTlHlFWN9wHeR2NnM5saBtQH1LU0wUXHRnUUvxF4G9WaTttJ+jpwDrC/pJ9TLYOxi+1baqtwNNbIcz7ADVQPNP5U0suBE6jWCDu6LKmyBbDryIrKMUkNDd8JQNGRUcFnKaoAsxVVAPoU8Hrge8BrqZZMvzDNJdGOpDcAbyqv1wDfoQo2h0k6hWoV5KNs/7XGak55VVbTzAiUABQTGhV83g28CHgacLXtEyV9BPgA1S+SL9j+fn21jaZq/R4VawMn2b4SuFLSjVSZ0K62f15PLYfQgPt1upE+oJhQS/B5IfB84L3A0cBhkra1fQfwEaqmlNtrq2g0Wsv3aNOy60bgqS3HjwHmA0fk4eXeSh9QTDmjMp+tqfp8brJ9OXC5pLuAT0l6v+3TJB3a0rYf8W8krQZ8QtLPgM8Be0r6KHA8sAFwCfDpTN/UY8mAYioZFXxeR/XL4bfA6pKeL2mW7a8DXwM+mJkOokN3UWXQWwH7AC8BVqD64+YA4Jj0+fSauv5vUJIBxZhags9s4NW2X122PwzsXr3V+ba/JOk7tv9RY3WjgUb9EbMb8FvbV0iaDxwGfBR4yPZbyjkr2L6nvhoPr/QBxZRSHhTcEDgWeFDSY8uhI6n6evYFNgPIL40YbVTweQawJXCSpGeVqXSuAn5BlT2/q1z293pqO9z6sBxQzyQAxb+MMRvxZVRDrNcBNpa0pO1/Ah8G/kimRIlxtASf7YHP2X471R8z35e0Qfke3QJ8Czip9Zrog4ZGoDTBxb+0/NLYA1iPajr1bwMLqJpMjpR0oe2HgA/VVtForFGZzz5Us6GPLKfw8fJHzjckXUI13c4Otv9SW4WniaY+B5QMKBYh6a1UHcJ3AesDp5XXXKpsaJP6ahdNViYWHQk+T6aaLfmFVDMjA2D7Y8B7gJ8AL7f9pzrqOt00dS64ZEDT3MhfrC1/uT4beLvtC8vxQ4BP2H6jpBXI0sgxvpeUZ3z+CLze9n+UZ8dOl/SXMmpyZMHCGKBm5j/JgKa1UU+mrydpCaqn07duOe0UyvfE9tFpLonx2P421bIc/wO8ruw7l2rmjI+X7DoGrcvsJ7NhR9+Naqs/APgp1WwGlwFvL/N0QZURrSNpxdHLMESM8Z34InAh1WwGywDY/jXwcuCgfI/q0sxRCAlA01RL8HkF8BzgP6lGtf2danjshyR9gWpZ7bfZvjujlKLVqD9iXippO2Cu7e2pZrSeW47tDRhYP9+jwRPNzYDSBzSNlQXBvgj8wva1ZUmFkaWz/wbMoVqX5Y666hjNUzIYjUy7VEa7HUQ1cOVKSV+xvbOkH0n6MdVgllfbfri+Wk9vTU05kwFNY2VBsHcCO0javQyv/j7V8tmPAHcm+MQYZrYEn5cCr6Jqqt2WahntfSQ92/ZOwFHANmXG66hJUzOgBKBpzvYPqJZROKQEoUeA44CP2r677cUx7ZTJRH9QZsqYBcymGmq9WclwPgfcB7xN0mZlOfabaqxy0P1scIOSJrjA9qmSHgHmSFpg+0SqheYiFmH7Nkm7U41su8j2kZKWBA6WdLjty0vf4X8Df66zrtGioW1wyYACANs/A95AtR5LxLjKxLOPpervWQE4FLgIOFzSxmUZ9g/ZvrXOesajmjkGLgEoWtg+I8toRyds/xB4I3AxsDzVzNZXUw21fgzVqLdogG77fzIKLiIaz/bPJb0D+A3wPNuHSlqlDGaJBmnqXHAJQBExabZ/WvqAzpQ0O6MmG6qZ8ScBKCIWj+0fSvpFHjBtrobGnwSgiFh8tu+ruw4xvqZOfpQAFBEx1Ab7bE83EoAiIobYyFxwTZRh2BERUYtkQBERQy4ZUESfSFoo6VJJV0r635F1aCZZ1taSTinvXyHp4DbnrijpLZO4xxGS3t3p/lHnHCfpNV3cax1JmQh0mmvqXHAJQDEMHrC9ke0NgH8C+7ceLBNndv1dt/1j2x9rc8qKQNcBKGKgGjwTQgJQDJtfAU8tf/lfLelLVPPbPUHS9pLOlzS/ZErLAkjaQdLvJJ1LtbQAZf/ekr5Y3q8h6WRJl5XXFsDHgKeU7OuT5bz3SLpI0uWSjmwp6wOSfi/pF1Tr47Ql6U2lnMsknTQqq3uRpF9J+oOkl5XzZ0r6ZMu9/3tx/yFjOHQ7D1zmgouYhLI8wI7AFWXX+sA3bT8XuJ9q0swX2d6Yag6zAyUtBRxDtWT0VsDjxin+88A5tjcENgauAg4Gri3Z13skbQ+sB2wGbARsIukFkjYBdgeeSxXgNu3g4/zA9qblflcD+7YcW4dqCYSXAl8pn2Ff4B7bm5by3yRp3Q7uE9NBQyNQBiHEMFha0qXl/a+AY4HHA9fb/k3ZvznwTOC8akFPlgTOB54O/Mn2NQCSvg3sN8Y9tgX+C8D2QuAeSSuNOmf78rqkbC9LFZCWA04us0hTVgmdyAaSPkTVzLcscFrLsRPKuk3XSLqufIbtgee09A+tUO79hw7uFUMuzwFF9M8Dtjdq3VGCzP2tu4AzbL921Hkb0buZm0W1kN9XR93jnZO4x3HAzrYvk7Q3sHXLsdFludz7bbZbAxWS1unyvjGEMgouol6/AbaU9FQASctIehrwO2BdSU8p5712nOvPBN5crp0paXmqRfuWaznnNOANLX1La0laHfgl8EpJS0tajqq5byLLATdLWgLYY9SxXSTNKHV+MvD7cu83l/OR9DRJj+3gPjENNLQFLgEopgfbtwF7A9+TdDlVQHq67QepmtxOLYMQrh+niHcA20i6ApgHPKvM/HxeGf79SdunA98Fzi/nnQgsZ3s+cDxwKXASVTPhRD4IXACcQRUkW/0eOAf4GbB/+QxfA34LzC/Drr9KWjhiRB8iUBm883tJf2z3uELbMjKBbUTE8Np4k9k+7zcXd3XNMktqnu3Z4x2XNJOqf/HFwA1UK+K+1vZvu7lPMqCIiCE2Mhdcj58D2gz4o+3rbP8T+D6wU7d1S4oeETHE5s+fd9rSS2jVLi9bSlJr2jTH9pyW7bWAv7Zs3wD8R7d1SwCKiBhitnfoQ7Fj5Uld9+ekCS4iIrp1A/CElu21gZu6LSQBKCIiunURsJ6kdSUtSTXTRycPWC8iTXAREdEV2wskHUD1/NlM4Ou2r+q2nAzDjoiIWqQJLiIiapEAFBERtUgAioiIWiQARURELRKAIiKiFglAERFRiwSgiIioxf8HHPQZudptgp4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.98      0.99        44\n",
      "           1       0.98      1.00      0.99        56\n",
      "\n",
      "    accuracy                           0.99       100\n",
      "   macro avg       0.99      0.99      0.99       100\n",
      "weighted avg       0.99      0.99      0.99       100\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_true = np.argmax(y_test[0:100],axis=1)\n",
    "pred = model.predict(x_test[0:100])\n",
    "pred = np.argmax(pred,axis=1)\n",
    "\n",
    "cm = confusion_matrix(y_true, pred)\n",
    "print(cm)\n",
    "plt.figure()\n",
    "plot_confusion_matrix(cm, ['normal','intruder'])\n",
    "plt.show()\n",
    "print(classification_report(y_true, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score (RMSE): 0.1\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD4CAYAAADhNOGaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZhU9b3n8fe3qkFiBImCN2yGzgwmEoMbLolbYlzA8UJMzIhLojGKElAT88zATSbLNT43y01cx+iFGLcYUUhUboaMTgxZFHWaDoogQbmK2kHHDqhREek65zt/1KmmuqnuPkAdqz2/z+t5eKg6dbr6d6q6+tPf3/cs5u6IiEi4Co0egIiINJaCQEQkcAoCEZHAKQhERAKnIBARCVxTowewvYYNG+Zjx45t9DBERN5VWltb/+buw2s99q4LgrFjx7Js2bJGD0NE5F3FzJ7r6TFNDYmIBE5BICISOAWBiEjgFAQiIoFTEIiIBC6zIDCzn5nZy2a2sofHzcyuMbO1ZrbCzA7KaiwiItKzLCuCm4FJvTw+GRiX/JsOXJ/hWEREpAeZHUfg7n80s7G9rDIVuNXL58F+xMyGmtkId38xqzEF6an74R/Gw+6jGz2SfisqlWi581/grVcbPRSRXu1x0FT2OeiYuj9vIw8oGwW8UHW/LVm2TRCY2XTKVQN77733OzK43LjrC3D4DDju240eSb/13F9aOfzpHwMQuzV4NCI9axkyAnIWBLU+cTWvkuPuc4G5ABMnTtSVdLZHaTNEWxo9in6ttGUzAI8f/W/sf+y0Bo9GpGeHZfS8jdxrqA0YU3V/NLC+QWPJpzgGHOJSo0fSr8VRBwBm77ozrojURSODYBHwhWTvocOB19QfqLNKACgIehVH5dfHigMaPBKRxsjsTyAzuwP4BDDMzNqAbwMDANz9BmAxcBKwFtgEfDGrsQRLQZBKpSIoFFURSJiy3Gvo9D4ed2BmVt9fAI/K/8dRY8fRz3kUA1BoUhBImHRkcZ51VgQKgt54pUegikACpSDIs0oAaGqoV3Hy+hTVI5BAKQjyTD2CVDxpFqtHIKFSEOSZgiCVytRQoUkVgYRJQZBn6hGkUpkaKmhqSAKlIMgz9QjSiSo9Ak0NSZgUBHmmqaFUKgeUaWpIQqUgyDMFQTqVvYZ0HIEESkGQZ+oRpFLZa6ioikACpSDIM1UE6eg4AgmcgiDP1CxOxWP1CCRsCoI8U0WQTvL6NKlHIIFSEOSZegSpqEcgoVMQ5JkqgnQ6KwIFgYRJQZBn6hGkk7xOOqBMQqUgyDNVBOnEJUpewAr6OEiY9JOfZ+oRpBNHRBQbPQqRhlEQ5JkqglQs7iDSR0ECpp/+PFOPIB2PKJkqAgmXgiDPVBGkYnFJU0MSNAVBnqlHkI6rRyBhUxDkmSqCVCwuEeujIAHTT3+eKQhSMe01JIFTEOSZmsWpmJeI1CyWgCkI8qwSAB6Be2PH0o+ZR8QKAgmYgiDPqisBNYx7ZHFErKkhCZiCIM+6BIGmh3qiqSEJnYIgz6qrAAVBjwquikDCpiDIM1UEqahHIKFTEOSZegSpFLykIJCgKQjyTBVBKqoIJHSZBoGZTTKzNWa21szm1Hh8bzNbYmbLzWyFmZ2U5XiCox5BKgWPcAWBBCyzIDCzInAdMBkYD5xuZuO7rfY/gLvc/UBgGvCTrMYTJFUEqRQ8IjZdnUzClWVFcCiw1t2fcfctwHxgard1HBiS3N4dWJ/heMKjIEhFFYGELssgGAW8UHW/LVlW7TvAWWbWBiwGLqr1RGY23cyWmdmy9vb2LMaaT2oWp1JAPQIJW5ZBYDWWdT/PwenAze4+GjgJuM3MthmTu89194nuPnH48OEZDDWnVBGkUvQI19SQBCzLIGgDxlTdH822Uz9fAu4CcPeHgUHAsAzHFBY1i1MpEOEFVQQSriyDoAUYZ2bNZjaQcjN4Ubd1ngc+BWBm+1IOAs391IsqglTUI5DQZRYE7l4CZgH3Aasp7x20yswuM7MpyWpfA843s8eBO4Bz3HWazLpRjyAVTQ1J6DL96Xf3xZSbwNXLvlV1+0ngiCzHEDRVBKkUNTUkgdORxXmmHkEqBWK8oIpAwqUgyDNVBKkUiUA9AgmYgiDP1CNIpTw1pIpAwqUgyLO4BE2Dtt6WmppcFYGETUGQZ3EETbsktxUEPSkS44UBjR6GSMMoCPJMFUEqRSLQXkMSMAVBnsUlVQR98DimyWJQj0ACpiDIsy4VgZrFtURREpCqCCRgCoI8iyMoqiLoTanUUb6hikACpiDIs7gETQO33pZtREkQmCoCCZiCIM/ULO5TqVSZGlJFIOFSEORZ9e6jHjd2LP2URwoCEQVBnsWRKoI+VHoEVlQQSLgUBHmm3Uf7FCcVgakikIApCPJMPYI+RaUt5RuqCCRgCoI8U0XQJ1UEIgqCfOtyHIEOKKslUo9AREGQa3EJigPACqoIerC1ItBJ5yRcCoI8i0vl3SILTQqCHkTJcQSqCCRkCoI8UxD0KY7KU0MFBYEETEGQV+7gUVUQqEdQS6wegYiCILcqv/gLTeUza6oiqKlyZLEqAgmZgiCv4qrTK2tqqEeV01CrIpCQKQjyKq46h46CoEfe2SPQXkMSLgVBXm0TBOoR1BJrakhEQZBb6hGk4rEqAhEFQV6pR5BKXFJFIKIgyCv1CFLx5HUpNikIJFwKgrxSjyCVrbuPampIwqUgyKsuQaAeQU86g6BJQSDhyjQIzGySma0xs7VmNqeHdf6rmT1pZqvM7BdZjiconc1i9Qh60zk1pIpAApbZxKiZFYHrgOOBNqDFzBa5+5NV64wD/gk4wt1fMbO9shpPcNQjSGVrRaAegYQry4rgUGCtuz/j7luA+cDUbuucD1zn7q8AuPvLGY4nLOoRpFKpCJpUEUjAsgyCUcALVffbkmXV9gH2MbOHzOwRM5tU64nMbLqZLTOzZe3t7RkNN2fUI0gnVkUgkmUQWI1l3u1+EzAO+ARwOvBTMxu6zRe5z3X3ie4+cfjw4XUfaC51OaBMU0M9qUwNFZsGNngkIo2TZRC0AWOq7o8G1tdY515373D3Z4E1lINBdpYOKEtHxxGIZBoELcA4M2s2s4HANGBRt3XuAT4JYGbDKE8VPZPhmMKhZnEqnlROTdp9VAKWWRC4ewmYBdwHrAbucvdVZnaZmU1JVrsP2GBmTwJLgP/m7huyGlNQtukRqFlcU3KuoaKCQAKWaT3s7ouBxd2WfavqtgOXJv+kntQjSCd5nYo615AELFVFYGaXpFkm/Yh6BOnEJSI3CsVio0ci0jBpp4bOrrHsnDqOQ+pNPYJ04hIRCgEJW6/1sJmdDpwBNJtZdaN3MKC5/P5MPYJULI4oUUQ7j0rI+poYXQq8CAwDfly1/HVgRVaDkjpQRZBOXCLSuRclcL0Ggbs/BzwHfOydGY7UjU46l4rFJSLT1JCELdWuEmb2OluPCh4IDADedPchWQ1MdpIqgnQ8Uo9AgpcqCNx9cPV9M/s05ZPKSX+lk86lYmoWi+zY5Ki73wMcW+exSD3ppHOpmEfECgIJXNqpoc9U3S0AE9n2BHLSn+iAslTKPQI1iyVsaQ+n/Meq2yVgHdteW0D6k1oHlLmD1TopbLhUEYik7xF8MeuBSJ117xEAeAzaQ6YLc+01JJL2FBMfNLN/N7N2M3vZzO41sw9mPTjZCd17BNXLpJPFqghE0k6O/gK4CxgBjAQWAHdkNSipg+49AlAQ1GAeEasikMClDQJz99vcvZT8+zlqFvdv3XsE1cukU0FBIJK6WbzEzOZQvgC9A6cB/8vM9gBw940ZjU92VFwq9wPMqoJAxxJ0Z14iNp2CWsKW9hNwWvL/Bd2Wn0s5GNQv6G/i0tYAUI+gR6oIRNIHwb7uvrl6gZkN6r5M+pEuQaCpoZ4UPCIq6OpkEra0PYKlKZdJfxFHCoIUVBGI9H09gvcDo4D3mNmBQOVopCHArhmPTXZGXNo6JaQg6FGBCFcQSOD6mho6kfKVyEYDV1Qtfx34ekZjknqoOTWkZnF35YpAzWIJW1/XI7gFuMXMPuvuv3yHxiT1oGZxKgVXRSCS9k+h/czsI90XuvtldR6P1It6BKloakgkfRC8UXV7EHAysLr+w5G6iUtQSPYFMFUEPSmqIhBJfdK56usVY2Y/Ahb1sLr0B+oRpFIgwgsKAgnbjp6IfVd0EFn/ph5BKuWKQM1iCVvaC9M8wdZzCxWAvYDvZjUoqQOPVRGkUCDeGpQigUr7p9DJwPuAo4ChwGJ3b81sVLLzdBxBKkU1i0VSTw1NBW4DhgEDgJvM7KLMRiU7T6eYSKVI1d5VIoFK+wk4Dzjc3d8EMLMfAA8D12Y1MNlJahanUvQYVxBI4FJfjwCo/i0SsfV0E9IfdTmOQM3inhSJdPlOCV7aP4VuAh41s7uT+58GbsxmSFIXcQmKyVk1NTXUoyKRKgIJXqqKwN2vAL4IbAReAb7o7lf19XVmNsnM1pjZ2uTCNj2td6qZuZlNTDtw6YN6BKk0EWMKAglc6k+Au/8Z+HPa9c2sCFwHHA+0AS1mtsjdn+y23mDgYuDRtM8tKahH0Kc4iiiYqyKQ4O3oAWVpHAqsdfdn3H0L5ctcTq2x3neBHwK6yE096YCyPpVKHQCqCCR4WQbBKOCFqvttybJOyTUOxrj7r3t7IjObbmbLzGxZe3t7/UeaR3Gk4wj6ECVBoAPKJHRZBkGtvYq880GzAnAl8LW+nsjd57r7RHefOHz48DoOMcfUI+hTpSKgqIpAwpZlELQBY6rujwbWV90fDOwH/N7M1gGHA4vUMK4TBUGf4s6KQEEgYcsyCFqAcWbWbGYDgWlUnbHU3V9z92HuPtbdxwKPAFPcfVmGYwpHzR6BmsXV1CMQKcssCNy9BMwC7qN87YK73H2VmV1mZlOy+r6SUI+gT3GUvB4KAglcpp8Ad18MLO627Fs9rPuJLMcSHE0N9anSLC6oRyCBy3JqSBpJQdCnqKSKQAQUBPmlA8r6FEdbADBVBBI4BUFedTnpXAEwVQTdxElFoCCQ0CkI8qr6wjRQDgUFQRdR0ixWj0BCpyDIq+qpIVAQ1BBHld1HBzR4JCKNpSDII/cegkA9gmqV3Uc1NSShUxDkkcfl/7sEQVEVQTexdh8VARQE+VT5ha8eQa9UEYiUKQjyKK6xf7yCYBuVHkGhqB6BhE1BkEc9BoF6BNVcew2JAAqCfKr8wlePoFeVqaFCkyoCCZuCII/UI0hFFYFImYIgj9QjSMVj9QhEQEGQTwqCVCpTQ0VVBBI4BUEe1QyCoprF3XRODTUpCCRsCoI8qtksVkXQ3dYewcAGj0SksRQEeaRmcTrJ61FURSCBUxDkkXoEqXT2CLT7qAROQZBH6hGkE2v3URFQEOSTegSpePJ6NKkikMApCPJIPYJ0Yh1ZLAIKgnxSjyCdSBWBCCgI8kk9gnSS10PNYgmdgiCPVBGkoh6BSJmCII86m8XqEfTG4hKxG4Vise+VRXJMQZBHqghS8bhESR8BEX0KckkXpknF4hIRqgZEFAR51GOzWBVBFx4pCERQEOSTDihLxeISkekjIKJPQR7pgLJ0NDUkAmQcBGY2yczWmNlaM5tT4/FLzexJM1thZg+Y2QeyHE8w1CNIxTQ1JAJkGARmVgSuAyYD44HTzWx8t9WWAxPdfQKwEPhhVuMJinoE6cQKAhHItiI4FFjr7s+4+xZgPjC1egV3X+Lum5K7jwCjMxxPONQjSMW8RGwKApEsg2AU8ELV/bZkWU++BPym1gNmNt3MlpnZsvb29joOMafUI0jFVBGIANkGgdVY5jVXNDsLmAj8a63H3X2uu09094nDhw+v4xBzqvILv/qv3UIRcIjjhgypPyqoIhABIMsrcrQBY6rujwbWd1/JzI4DvgEc4+5vZziecPTUI6g8VtA1eqHcLI6145xIpp+CFmCcmTWb2UBgGrCoegUzOxD4N2CKu7+c4VjC0tNeQ9WPSTkIVBGIZBcE7l4CZgH3AauBu9x9lZldZmZTktX+FdgNWGBmj5nZoh6eTrZHZfqnVhC4diGtUBCIlGV6sVZ3Xwws7rbsW1W3j8vy+wcrLgEGhaqcV0WwjfLUkIJARBOkeRSXulYDUBUEqggqCqoIRAAFQT7VDIKqZrEACgKRCgVBHsVRLxWBgqBCQSBSpiDIo7jU9WAyUBDUUPASbpm2yUTeFRQEeaQeQSqqCETKFAR5pB5BKgWPcAWBSLa7j0qDqEeQSoEI7z6FJu+ojo4O2tra2Lx5c6OHkhuDBg1i9OjRDBgwIPXXKAjySD2CVIoeqUfQYG1tbQwePJixY8diVuv0ZLI93J0NGzbQ1tZGc3Nz6q/T1FAe9dojUBBUFNDUUKNt3ryZPffcUyFQJ2bGnnvuud0VloIgj3rtEahZXFH0GO/+Osk7TiFQXzvyeioI8kgVQSqqCETKFAR5FEfqEaRQJFJFIO+4devW8Ytf/GK7v+6cc85h4cKFGYxIQZBPqghSKRJ1vXiPyDtgR4MgS/pzKI90QFkqRVdF0J/887+v4sn1f6/rc44fOYRv/+NHel3n5z//Oddccw1btmzhsMMO4+tf/zrHHXccDz/8MHvssQfHHHMM3/zmN9lnn32YNGkShx12GMuXL2efffbh1ltvZdddd6W1tZVLL72UN954g2HDhnHzzTczYsQI1q5dy4UXXkh7ezvFYpEFCxYwZ84cVq9ezQEHHMDZZ5/NxRdfzJw5c/j973/P22+/zcyZM7ngggtwdy666CJ+97vf0dzcjHvNCzzWhSqCPNIBZakUibd9nSQoq1ev5s477+Shhx7iscceo1gs8oc//IHZs2dz4YUX8uMf/5jx48dzwgknALBmzRqmT5/OihUrGDJkCD/5yU/o6OjgoosuYuHChbS2tnLuuefyjW98A4AzzzyTmTNn8vjjj7N06VJGjBjB97//fY466igee+wxvvrVr3LjjTey++6709LSQktLC/PmzePZZ5/l7rvvZs2aNTzxxBPMmzePpUuXZvY66FOQR+oRpNKkA8r6lb7+cs/CAw88QGtrK4cccggAb731FnvttRff+c53WLBgATfccAOPPfZY5/pjxozhiCOOAOCss87immuuYdKkSaxcuZLjjz8egCiKGDFiBK+//jp//etfOeWUU4DygV613H///axYsaJz/v+1117j6aef5o9//COnn346xWKRkSNHcuyxx2b2OigI8iguQdMuXZcpCLZRpMYR2BIUd+fss8/me9/7XpflmzZtoq2tDYA33niDwYMHA9vummlmuDsf+chHePjhh7s89ve/p5vmcneuvfZaTjzxxC7LFy9e/I7tWqupoTxSj6BPcRRRNFcQBO5Tn/oUCxcu5OWXy5dM37hxI8899xyzZ8/mzDPP5LLLLuP888/vXP/555/v/IV/xx13cOSRR/KhD32I9vb2zuUdHR2sWrWKIUOGMHr0aO655x4A3n77bTZt2sTgwYN5/fXXO5/zxBNP5Prrr6ejowOAp556ijfffJOjjz6a+fPnE0URL774IkuWLMnsdVAQ5JF6BH2KouR10NRQ0MaPH8/ll1/OCSecwIQJEzj++ONZt24dLS0tnWEwcOBAbrrpJgD23XdfbrnlFiZMmMDGjRuZMWMGAwcOZOHChcyePZv999+fAw44oHM+/7bbbuOaa65hwoQJfPzjH+ell15iwoQJNDU1sf/++3PllVdy3nnnMX78eA466CD2228/LrjgAkqlEqeccgrjxo3jox/9KDNmzOCYY47J7HWwLDvRWZg4caIvW7as0cPo337ycdijGabdvnXZq8/DVR+FqdfBgWc1bmz9xOZNbzDoh6N4+IMX87EvfLfRwwnW6tWr2XfffRs9jFTWrVvHySefzMqVKxs9lD7Vel3NrNXdJ9ZaXxVBHuk4gj6VSuUy3FQRiCgIcklB0KeoVJkaUo9A0hk7duy7ohrYEQqCPFKzuE9RaQsAVlQQiCgI8qjmhWnULK4WqyIQ6aQgyCNdmKZPpajSI1AQiCgI8kg9gj5VKgJNDYkoCPJJPYI+RaoIJAO77bYbAOvXr+fUU0/tdd2rrrqKTZs2dd4/6aSTePXVVzMdX08UBHlUq0dgyVutigCAuLL7qCoC6UMUbf8fTyNHjuzz2gHdg2Dx4sUMHTp0u79XPehTkEe1egRm5XBQEAAQRwqCfuc3c+ClJ+r7nO//KEz+fo8Pr1u3ruappcePH8+5557L/fffz6xZszjkkEOYOXMm7e3t7LrrrsybN48Pf/jDPPvss5xxxhmUSiUmTZrU5XkrB59FUcTs2bO57777MDPOP/983J3169fzyU9+kmHDhrFkyRLGjh3LsmXLGDZsGFdccQU/+9nPADjvvPP4yle+wrp165g8eTJHHnkkS5cuZdSoUdx777285z3v2emXSRVBHtWaGgIFQZXKcQSaGpJap5aG8tlCH3zwQaZNm8b06dO59tpraW1t5Uc/+hFf/vKXAbjkkkuYMWMGLS0tvP/976/5/HPnzuXZZ59l+fLlrFixgjPPPJOLL76YkSNHsmTJkm3OIdTa2spNN93Eo48+yiOPPMK8efNYvnw5AE8//TQzZ85k1apVDB06lF/+8pd1eQ30KcijXoNAPQKAODnXUKFJH4F+o5e/3LNU69TSAKeddhpQPvvo0qVL+dznPtf5NW+//TYADz30UOcv489//vPMnj17m+f/7W9/y4UXXkhT8rO2xx579DqeBx98kFNOOYX3vve9AHzmM5/hT3/6E1OmTKG5uZkDDjgAgIMPPph169bt6GZ3kemnwMwmAVcDReCn7v79bo/vAtwKHAxsAE5z93VZjin34hjo4ayahaIqgkTn1FBhQINHIo1W69TSQOcv4jiOGTp0aJfrEvT29d25+3adTrq387/tssvW08sXi0Xeeuut1M/bm8ymhsysCFwHTAbGA6eb2fhuq30JeMXd/zNwJfCDrMYTjLiXs2pqaqhTZ0WgHkHwap1autqQIUNobm5mwYIFQPkX9eOPPw7AEUccwfz58wG4/fbbqeWEE07ghhtuoJRMR27cuBFgm9NRVxx99NHcc889bNq0iTfffJO7776bo446qg5b2rMsPwWHAmvd/RkAM5sPTAWerFpnKvCd5PZC4H+amXkGp0Rt+dXVDF85r95P2+8UcPYGfvrQ89y57A9dHpv/VszAljvY0HpfYwbXj7wv3gyoRyBbTy19wQUXMG7cOGbMmMG1117bZZ3bb7+dGTNmcPnll9PR0cG0adPYf//9ufrqqznjjDO4+uqr+exnP1vz+c877zyeeuopJkyYwIABAzj//POZNWsW06dPZ/LkyYwYMaJLn+Cggw7inHPO4dBDD+38+gMPPLBu00C1ZHYaajM7FZjk7ucl9z8PHObus6rWWZms05bc/49knb91e67pwHSAvffe++Dnnntuu8ez/P6f4yvu3NHNeVeJKPKbPb/ASwM/0GX5J19ZyH/anM+TZu2IUtNu7PvF6xi8e+9ztpKdRp+G+t10auntsb2noc7yz6Fak2LdUyfNOrj7XGAulK9HsCODOfCEs+CEcM7Df0jNpQe/w6MQkXeDLHcfbQPGVN0fDazvaR0zawJ2BzZmOCYRkU55PrX09sgyCFqAcWbWbGYDgWnAom7rLALOTm6fCvwui/6AiPRf+sjX1468npkFgbuXgFnAfcBq4C53X2Vml5nZlGS1G4E9zWwtcCkwJ6vxiEj/M2jQIDZs2KAwqBN3Z8OGDQwaNGi7vk7XLBaRhuno6KCtrY3Nmzc3eii5MWjQIEaPHs2AAV2PkWlUs1hEpFcDBgygubm50cMIns41JCISOAWBiEjgFAQiIoF71zWLzawd2P5Di8uGAX/rc638CXG7Q9xmCHO7Q9xm2P7t/oC7D6/1wLsuCHaGmS3rqWueZyFud4jbDGFud4jbDPXdbk0NiYgETkEgIhK40IJgbqMH0CAhbneI2wxhbneI2wx13O6gegQiIrKt0CoCERHpRkEgIhK4YILAzCaZ2RozW2tmuTzLqZmNMbMlZrbazFaZ2SXJ8j3M7P+Y2dPJ/+9r9FjrzcyKZrbczH6d3G82s0eTbb4zORV6rpjZUDNbaGZ/Sd7zjwXyXn81+fleaWZ3mNmgvL3fZvYzM3s5uYpjZVnN99bKrkl+t60ws4O29/sFEQRmVgSuAyYD44HTzWx8Y0eViRLwNXffFzgcmJls5xzgAXcfBzxAPk/3fQnl051X/AC4MtnmV4AvNWRU2boa+N/u/mFgf8rbn+v32sxGARcDE919P6BI+VoneXu/bwYmdVvW03s7GRiX/JsOXL+93yyIIAAOBda6+zPuvgWYD0xt8Jjqzt1fdPc/J7dfp/yLYRTlbb0lWe0W4NONGWE2zGw08F+Anyb3DTgWWJisksdtHgIcTfmaHrj7Fnd/lZy/14km4D3JVQ13BV4kZ++3u/+Rba/W2NN7OxW41cseAYaa2Yjt+X6hBMEo4IWq+23Jstwys7HAgcCjwD+4+4tQDgtgr8aNLBNXAf8diJP7ewKvJhdHgny+3x8E2oGbkimxn5rZe8n5e+3ufwV+BDxPOQBeA1rJ//sNPb+3O/37LZQgsBrLcrvfrJntBvwS+Iq7/73R48mSmZ0MvOzurdWLa6yat/e7CTgIuN7dDwTeJGfTQLUk8+JTgWZgJPBeylMj3eXt/e7NTv+8hxIEbcCYqvujgfUNGkumzGwA5RC43d1/lSz+f5VSMfn/5UaNLwNHAFPMbB3lKb9jKVcIQ5OpA8jn+90GtLn7o8n9hZSDIc/vNcBxwLPu3u7uHcCvgI+T//cben5vd/r3WyhB0AKMS/YsGEi5ubSowWOqu2Ru/EZgtbtfUfXQIuDs5PbZwL3v9Niy4u7/5O6j3X0s5ff1d+5+JrAEODVZLVfbDODuLwEvmNmHkkWfAp4kx+914nngcDPbNfl5r2x3rt/vRE/v7SLgC8neQ4cDr1WmkFJz9yD+AScBTwH/AXyj0ePJaBuPpFwSrgAeS/6dRHnO/AHg6eT/PRo91oy2/xPAr5PbHwT+L7AWWADs0ujxZbC9BwDLkvf7HuB9IbzXwD8Df6HEc0MAAABZSURBVAFWArcBu+Tt/QbuoNwD6aD8F/+XenpvKU8NXZf8bnuC8h5V2/X9dIoJEZHAhTI1JCIiPVAQiIgETkEgIhI4BYGISOAUBCIigVMQiIgETkEgIhK4/w9eTWTAzys7kgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reg_score2 = np.sqrt(metrics.mean_squared_error(pred,y_true))\n",
    "print(\"Score (RMSE): {}\".format(reg_score2))\n",
    "\n",
    "# Plot the chart\n",
    "chart_regression(pred, y_true, sort=True)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}