ConvNetTest-MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized TensorFlow\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "# loading keras and FBD experiment...\n",
    "from keras import backend as K\n",
    "from helpers import *\n",
    "from experiment_model import *\n",
    "from model_conv import *\n",
    "from matplotlib import pyplot as plt\n",
    "import pickle\n",
    "from tqdm import tqdm\n",
    "import sys\n",
    "import numpy as np\n",
    "\n",
    "# to show the model\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "from keras import Model, Input\n",
    "\n",
    "# loading resnet\n",
    "from keras.applications.resnet50 import ResNet50\n",
    "from keras.preprocessing import image\n",
    "from keras.applications.resnet50 import preprocess_input, decode_predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The issue:\n",
    "1. MNIST convnet works with avgpool/maxpool and with 3 channels\n",
    "2. ImageNet doesn't work even for one conv layer and downsampled to 2x2 pixels input (see VGG16 notebook) both for vgg16 and resnet\n",
    "3. Looks like the problem is in `preprocess_input`. It makes $\\mathbb{E}_ix_i=0$. Tried removing it. VGG still works and the bound still doesn't. Maybe the problem is that it's rare that a pixel becomes of different color? Maybe I need to kill all 3 channels at once? Or I need to use AvgPooling? Even 1 Conv layer with flattened output cannot be handled by the bound\n",
    "4. Which assumption is not correct? Is it the issue with input dimensionality? Try only crashing first row of the image..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# see https://github.com/keras-team/keras/blob/master/examples/mnist_cnn.py\n",
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D, AveragePooling2D\n",
    "from keras import backend as K\n",
    "\n",
    "batch_size = 128\n",
    "num_classes = 10\n",
    "epochs = 10\n",
    "\n",
    "# input image dimensions\n",
    "img_rows, img_cols = 28, 28\n",
    "\n",
    "# the data, split between train and test sets\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# channels last\n",
    "# Making images grayscale (adding repeated 2 more channels)\n",
    "x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1).repeat(3, axis = 3)\n",
    "x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1).repeat(3, axis = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((10000, 28, 28, 3), (60000, 28, 28, 3))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# printing shapes\n",
    "x_test.shape, x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (60000, 28, 28, 3)\n",
      "60000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "input_shape = (img_rows, img_cols, 3)\n",
    "\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# convert class vectors to binary class matrices\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "60000/60000 [==============================] - 6s 94us/step - loss: 2.3208 - acc: 0.1084 - val_loss: 2.3008 - val_acc: 0.1135\n",
      "Epoch 2/10\n",
      "60000/60000 [==============================] - 5s 86us/step - loss: 1.7512 - acc: 0.3403 - val_loss: 0.1380 - val_acc: 0.9565\n",
      "Epoch 3/10\n",
      "60000/60000 [==============================] - 5s 79us/step - loss: 0.1497 - acc: 0.9553 - val_loss: 0.0550 - val_acc: 0.9822\n",
      "Epoch 4/10\n",
      "60000/60000 [==============================] - 5s 90us/step - loss: 0.0943 - acc: 0.9725 - val_loss: 0.0374 - val_acc: 0.9871\n",
      "Epoch 5/10\n",
      "60000/60000 [==============================] - 5s 87us/step - loss: 0.0723 - acc: 0.9791 - val_loss: 0.0297 - val_acc: 0.9906\n",
      "Epoch 6/10\n",
      "60000/60000 [==============================] - 5s 79us/step - loss: 0.0625 - acc: 0.9813 - val_loss: 0.0277 - val_acc: 0.9894\n",
      "Epoch 7/10\n",
      "60000/60000 [==============================] - 5s 79us/step - loss: 0.0544 - acc: 0.9841 - val_loss: 0.0258 - val_acc: 0.9910\n",
      "Epoch 8/10\n",
      "60000/60000 [==============================] - 5s 83us/step - loss: 0.0485 - acc: 0.9860 - val_loss: 0.0297 - val_acc: 0.9909\n",
      "Epoch 9/10\n",
      "60000/60000 [==============================] - 5s 85us/step - loss: 0.0463 - acc: 0.9865 - val_loss: 0.0275 - val_acc: 0.9908\n",
      "Epoch 10/10\n",
      "60000/60000 [==============================] - 5s 82us/step - loss: 0.0431 - acc: 0.9877 - val_loss: 0.0235 - val_acc: 0.9919\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xt0m/d93/H3FwDvkkhJpGxZknVJdPUltS3LarLGSZOudpLZXdJt8Vm6Ne3ibqfOsjZr4lxOmqa3tel62p5lab2ul3RdMs9JetTWm3vSpidZE9GWc5FJXWyZ1F0GKUIiJd4BfPfHA5AgRYogCTwPAXxe5+AAePAQv68g6aNH3+eH32PujoiIVJdY1AWIiEjpKdxFRKqQwl1EpAop3EVEqpDCXUSkCincRUSqkMJdRKQKKdxFRKqQwl1EpAolohq4vb3dt23bFtXwIiIV6cUXX7zs7h0L7RdZuG/bto0jR45ENbyISEUyszPF7Ke2jIhIFVK4i4hUIYW7iEgVWjDczeyPzKzPzLrmed3M7PfM7JSZHTWze0tfpoiILEYxR+5/Ajx0k9cfBnbmbo8Dn19+WSIishwLhru7fwNI3WSXR4EveOAw0GZmG0tVoIiILF4peu6bgHMFz8/ntt3AzB43syNmdqS/v78EQ4uIyFxCnefu7k8BTwHs379f1/eTxclmITORu01OP86mZ27HwAwsBrF4cH/DzebZvoR9iuUO2Qx4Fjwz63Hu3rO57YWPs/P83Fz75t8r/37z/DWbt+55ts+5/2L2vdn+8+y+0Is3/ezne81zn43nbtkbbzP2mf36TbYxz/vNNdbuh2DTfTf7hS9bKcL9ArCl4Pnm3DapVNkspMdm3cZhcjS4n709PQaTY3MHb/5xdrJg+xyv3/A4feM2z0T9yczN5voHxG4MXnQ8I5DFODOxhu0VEO6HgCfM7EvAA8Cgu18qwfvKzbjDtUvQdxyGLxcEbWEA3ySQbxbUmYnl1xerg3j+Vp+7zfO4YXVwH0ssvO+Mx3PsH6vLfT6FR68LHXkVc3Q2fctmg5tnM3g2M/U4m83g2SzuWbKZLFmLkSVGhhhZN7LESBMji5HxGJmC+7THyHiMNEbWjUmPkXEjjZHOGhliTGaNtBfcssF+kw6T2Rhph8n8tmzwOJ2FjIO7k8k6Wc/dph4ztd09G/znaNbr+f0L2Tz/UM1/zD7f/kv7B+9mP3ez2rIYnrtlCX5PslPPYzjM2ja9D7nfm+zUvvnH0/s6MXzG//iMmMUL/hdoxGIxPt6+l+1L+pUXb8FwN7MvAm8B2s3sPPCLQB2Au/8+8CzwDuAUMAK8v1zF1qzr/dB/PAjywtv44Pw/E0tAorHg1hDc1+WeN66BRNP09kQD1BU+n+fnEg0zfy6/Pd4QbMsH7WLaFbO4O2OTWUYm0oxMZBidzDA8nmZ0IsPIRIaRyQyjE2lGxnPP8/vl9k1nnHQ2SyYbBFo6d1/4POtOOpO7v+H1LJksZLLZYN/C93Cft9NRTjGDRDxGImYkYkZdPEYibiRi+ftgWzxmJOIx6mJGoi54PRYz4gbxWIx4DOIxo86MeCx3yz2OFTzO32JmuZ+J5V7jJvvN3D69H5hZEI+5e8jlncHUFmNqn9zTGa9b7vXp3W3qj9nUe894Pr1z/mdjFtRqxlTNMQt+NlawLf96LL89Nv04v2/+1z1733z9UVsw3N39sQVed+BnS1ZRLRu9An0noO8Y9J+YDvGRy9P7NK2FDfvgrh+HDXuD2+qNN4ZxPLzTKaMTGS4NjnJtLM3IxDijk2mGxzO5ME7nwjgTbJtM5wI5t22iILQncqE9mVlUgMZjRnNdnKb64FYXj02FTCI+M3Qa6mI0WRCG+bBLxGIzgioRsxuex2IFP2PT75vI/QWfPc6MkI1PB3AiFqMubtNBHZ9jW/5nc+9ZlwtokcWIbOGwmjZ+DfpPBiFeGObXCrpZ9athwx7Y844gzDv2BPerNizrqHixJtJZkkNjXLw6yqXBMS4OjnLp6hiXBke5mLu/MjK54PuYQXNdnOaGBM31cZrq4jTXx2mpT9CxqiHYVh+8Ftxy+009j9NUl6ClIT69by7QGxKxFXO0JLJSKNzLaXI0F+LHC9oqJ2Dw7PQ+iSbo2A073hqEeT7IWzeXPcQzWaf/2vicgX1xcIxLV0fpvz5+w1F0a1MdG1sbua2tiXtub+O2tiY2tjbS2lSXC+MbQ1oBLBIuhXsppCdg4JXpNkp/7mg81cvUDIl4PbTvgi0H4L5/Pd1SadsaTNcrMXcnNTwRHG3Pc9SdHBojPetEWXN9fCq4d+/uYGNrE7e1NebugxBvadAfG5GVTn9Ll+rMt6Hz94MwT70azLWGYFrc+tfBrXfB3f9iup2ybkdZ+uCv9l+nsyc186g7F+bj6eyMfevjMW5tbWRjayMHtq+bCvGp8G5tYk1TQkfYIlVA4b5U3/zPcPbbsP1B2Puu6XZK+87gpGZI/s2fHqH38jAxg1vWBMF9x6ZW/vEdt7KxtXHGkff6lnqdmBOpEQr3pUp2w553wbv/ILISLg2O0nt5mF/40d38zJt3kIhrBWcRCSgNlmIkBdcuwi13RFpGZ0+wntuDuzoU7CIygxJhKZLdwX3U4d47wOrGBHs3rom0DhFZeRTuSzEV7ndGWkZnT4r7t60jrj66iMyicF+KZBc0rw++UBSRvqExei4P88D2dZHVICIrl8J9KZLdQUsmwimDz58O+u0P7FgfWQ0isnIp3Bcrmwm+pLQCWjIt9XHuvE39dhG5kcJ9sa6chsmRFXEy9b5t6zRLRkTmpGRYrGRXcB9huKeGJ3g5eV39dhGZl8J9sZLdwaL7HXsiK+H53gEADu5QuIvI3BTui5XshvWvDy5sEZHDPSka62LctaktshpEZGVTuC9WsmsF9NtT3Ld1LfUJ/faJyNyUDosxfi04oRphuA+OTHLitSEe2K4pkCIyP4X7YvQdD+4jnAb5/OkU7uhkqojclMJ9MVbATJnOngHqEzHesEX9dhGZn8J9MZLd0LAGWrdEVkJnb4of2NJGY13pr94kItVD4b4YyWPBRTkiWnZgaGyS7ouDHFRLRkQWoHAvlvv0mjIRefHMFbKu9WREZGEK92INnofxwYj77Snq4sa9t6+NrAYRqQwK92KtgDXcO3sHuHtzG0316reLyM0p3IuVnymzYW8kw49MpHnp/KCmQIpIURTuxUp2Q9tWaIxmid0Xz1whnXX120WkKAr3YiW7o23J9KSIx4z7tqrfLiILU7gXY3IMBl6J9mRq7wB3bmplVUMishpEpHIo3IvRfwI8G1m4j01m+P45zW8XkeIp3IsxNVMmmnD/ztkrTGSyPKD120WkSAr3YiS7IdEI63ZEMnxnTwoz2L9N4S4ixVG4F6OvO5gCGYtmfnln7wD7Nq5hTWNdJOOLSOVRuBcjwmUHxtMZvnv2qtZvF5FFUbgv5HofDPdHNg3y6PlBxtPqt4vI4ijcFxLxGu6dPcHFsA+o3y4ii1BUuJvZQ2Z20sxOmdmTc7x+u5l93cy+a2ZHzewdpS81IvmZMhsiCvfeFHtuXc3alvpIxheRyrRguJtZHPgc8DCwD3jMzPbN2u2TwNPufg/wXuC/lrrQyCS7YfVGaAm/5z2ZyfLimStaT0ZEFq2YI/cDwCl373H3CeBLwKOz9nEgv+hKK3CxdCVGLNkVWUvmpQuDjExktJ6MiCxaMeG+CThX8Px8bluhTwPvM7PzwLPAB+d6IzN73MyOmNmR/v7+JZQbsswk9J+MsN+eAuCAjtxFZJFKdUL1MeBP3H0z8A7gz8zshvd296fcfb+77+/o6CjR0GU0cAoyE5HNlOnsHeD1G1bRvqohkvFFpHIVE+4XgMIrQm/ObSv008DTAO7+baARaC9FgZGaOpk6+xRD+aUzWY6cVr9dRJammHB/AdhpZtvNrJ7ghOmhWfucBd4GYGZ7CcK9AvouC0h2QywB7btCH/rYpSGuj6fVbxeRJVkw3N09DTwBPAccJ5gV021mnzGzR3K7fRj4gJl9H/gi8JPu7uUqOjTJbmjfDYnwpyHm++06cheRpShqcXB3f5bgRGnhtk8VPD4GvKm0pa0AyW7Y+sZIhu7sHWDb+mZuWdMYyfgiUtn0DdX5jF6BofORzJTJZJ3ne1NaT0ZElkzhPp/kseA+gpkyJ1+7xtBYWuvJiMiSKdznE+EFOjp7g/VkdDJVRJZK4T6fZBc0rYPVt4Y+dGdPis1rm9jU1hT62CJSHRTu88mv4W4W6rDuzvOn1W8XkeVRuM8lm4W+Y5H021/pu05qeEL9dhFZFoX7XK70wuRINP323PrtB3XkLiLLoHCfy9TJ1PCXHTjcm2JjayNb1qnfLiJLp3CfS98xwKBjb6jDujudPSke2L4OC7nXLyLVReE+l2QXrH8d1DeHOmzP5WEuXx/XFEgRWTaF+1zyM2VCpvVkRKRUFO6zjV+HVG8kM2U6ewdoX9XA9vaW0McWkeqicJ+t/wTgoR+5T/Xbd6jfLiLLp3CfLdkV3Icc7mdTI7w2NMZBtWREpAQU7rMlu6F+NbTeHuqwnb25frtOpopICSjcZ0t2B/PbY+F+NJ09Kda11LNzw6pQxxWR6qRwL+QetGUiWgnywDb120WkNBTuhYYuwNhg6BfEvnB1lPNXRrWejIiUjMK90NSyA+FOg8yvJ6OVIEWkVBTuhSJaU6azJ0VrUx17bl0d6rgiUr0U7oWS3cEsmcbWUIft7B3g/m3riMXUbxeR0lC4F4pg2YHk0BinB0Y4qH67iJSQwj0vPQ6XXw493A+r3y4iZaBwz+s/CZ4JPdw7e1Osakiwd6P67SJSOgr3vAhnyuzftpZEXL8VIlI6SpS8ZBckGmHdjtCG7L82zqv9w2rJiEjJKdzzkt3QsQfiidCGfOF0fj0ZnUwVkdJSuOcluyNpyTTXx7lrU7hTL0Wk+incAa73w3Bf+F9e6k1x39a11KnfLiIlplQB6MufTA1vpsyV4QlOvHZNl9QTkbJQuEMkM2WeP63120WkfBTuEIT7qlugpT20ITt7UjQkYty9Wf12ESk9hTtEsoZ7Z+8A996+loZEPNRxRaQ2KNwzaeg7EWq4D45OcuzSkKZAikjZKNxTr0JmPNR++5HTKdy1noyIlE9R4W5mD5nZSTM7ZWZPzrPPPzezY2bWbWb/s7RlllGyK7gP8ci9szdFfTzGPbe3hTamiNSWBb+OaWZx4HPAjwDngRfM7JC7HyvYZyfwMeBN7n7FzDaUq+CSS3ZDLAHtu0IbsrNngDdsaaWxTv12ESmPYo7cDwCn3L3H3SeALwGPztrnA8Dn3P0KgLv3lbbMMkp2B8GeaAhluOvjabouDqklIyJlVUy4bwLOFTw/n9tWaBewy8z+wcwOm9lDpSqw7EK+QMeLZ66QybpOpopIWZXqhGoC2Am8BXgM+G9mdkND2cweN7MjZnakv7+/REMvw+hVGDwHG8JbdqCzZ4BEzLhv69rQxhSR2lNMuF8AthQ835zbVug8cMjdJ929F3iZIOxncPen3H2/u+/v6OhYas2l03c8uA9xpkxnb4q7NrfSXB/e6pMiUnuKCfcXgJ1mtt3M6oH3Aodm7fMXBEftmFk7QZump4R1lkfIM2VGJzIcPX9V/XYRKbsFw93d08ATwHPAceBpd+82s8+Y2SO53Z4DBszsGPB14BfcfaBcRZdMshsa22DNbaEM952zV5jMqN8uIuVXVG/A3Z8Fnp217VMFjx34+dytcuTXcDcLZbjOngFiBvvVbxeRMqvdb6hms9B3LNSZMod7U9y5qZXVjXWhjSkital2w/3qGZi4Hlq4j01m+N65q1q/XURCUbvhHvIa7t87d5WJdFYnU0UkFDUe7gYb9oQyXGdPCjO4f5uO3EWk/Go43Ltg3Q6obwlluM7eAfbcuobWZvXbRaT8ajjcu0O7IPZEOst3zl5Rv11EQlOb4T4xDKme0PrtL124ythkloOa3y4iIanNcO8/AXhoM2UO9wQXwz6gk6kiEpLaDPepmTLhhHtnb4pdt6xiXUt9KOOJiNRuuNe1QNu2sg+VzmR58XRKUyBFJFS1G+637INY+X/5XReHGJ7IaD0ZEQlV7YW7ezANMqyWTE+wftoBzZQRkRDVXrhfuwSjV0KbKdPZm2JHRwsbVjeGMp6ICNRiuId4MjWTdV7oVb9dRMJXg+Geu0BHCJfWO35piGvjac1vF5HQ1WC4d0PrFmi64RKvJXdY/XYRiUhthntIF8Tu7E1x+7pmNrY2hTKeiEhebYV7egIuvxxKvz2bdV44ndJ6MiISidoK98svQzYdSri/3HeNqyOTPLBDJ1NFJHy1Fe4hXqCjM7eejI7cRSQKNRbuXRCvh/WvL/tQnb0DbGprYsu65rKPJSIyW42Fezd07IF4oqzDuDvP96rfLiLRqb1wD6El82r/dS5fn9B6MiISmdoJ9+HLcP21UE6mHp7qt+tkqohEo3bCPcRlBzp7U9yypoGt69VvF5Fo1GC4l7ct4+509gzwwPb1mFlZxxIRmU9thXvLBljVUdZhTg+M0HdtXP12EYlUDYV7V3CBjjLLr9+ufruIRKk2wj2bCS6KHcaXl3pTtK+q53UdLWUfS0RkPrUR7qkeSI+V/WRqvt9+YPs69dtFJFK1Ee75NdzLHO7nr4xycXBMLRkRiVyNhHs3WBzad5d1mM7e3Px2nUwVkYjVTri374S68l7HtLNngLbmOnZtWF3WcUREFlIj4d4V2peXDmxbRyymfruIRKv6w31sEK6eLXu4Xxoc5WxqROu3i8iKUP3h3nc8uC/zNEit3y4iK0lR4W5mD5nZSTM7ZWZP3mS/95iZm9n+0pW4TCHNlOnsHWB1Y4K9G9eUdRwRkWIsGO5mFgc+BzwM7AMeM7MbvuppZquBDwGdpS5yWZLd0NAKazaVdZjOnqDfHle/XURWgGKO3A8Ap9y9x90ngC8Bj86x3y8DvwGMlbC+5UseC47ay/ilor6hMXouD2sKpIisGMWE+ybgXMHz87ltU8zsXmCLu/91CWtbPvfcBTrK3ZLR+u0isrIs+4SqmcWA3wY+XMS+j5vZETM70t/fv9yhF3b1LExcC6Xf3lIf547b1G8XkZWhmHC/AGwpeL45ty1vNXAn8Pdmdho4CBya66Squz/l7vvdfX9HR3mX3gVCW8O9syfFfdvWkYhX/+QjEakMxaTRC8BOM9tuZvXAe4FD+RfdfdDd2919m7tvAw4Dj7j7kbJUvBj5cN+wt2xDDFwf55W+65oCKSIryoLh7u5p4AngOeA48LS7d5vZZ8zskXIXuCzJLli7HRpWlW2IF04H/faDOpkqIitIopid3P1Z4NlZ2z41z75vWX5ZJRLCydTDPSka62LctamtrOOIiCxG9TaJJ0Yg9Wr5++29Ke7bupb6RPV+lCJSeao3kfpPgGfLeuQ+ODLJideGNAVSRFac6g33qZky5Qv350+ncNd6MiKy8lR3uNc1BydUy6SzZ4D6RIw3bFG/XURWluoN977uYApkrHy/xM7eFPdsaaOxLl62MURElqI6w90dXivvBTqGxibpvjio9dtFZEWqznC/noTRVFlnyrx4+gpZh4Pqt4vIClSd4R7CGu7fevUydXHjntvXlm0MEZGlqtJwzy87cMOy8yVxZmCYPzt8hrftuYWmevXbRWTlqd5wX7MJmkvfMnF3nvzyS9TFYnz6kfJfdFtEZCmqN9zL1JL54vPn+HbPAB9/515ubW0syxgiIstVfeGenoD+k2UJ90uDo/zas8d54+vW8977tyz8AyIiEam+cB94BbKTJZ8p4+584qtdZLLOf3r33VgZL9snIrJc1RfuZTqZeuj7F/m7E338xx/dze3rm0v63iIipVaF4d4FsTpo31myt7x8fZxPH+rmntvb+Mk3bivZ+4qIlEsVhvsx6NgD8bqSveWnD3UzPJ7hN99zN/GY2jEisvJVYbiXdqbM33S/xl8dvcQHf/j17LxldcneV0SknKor3EdScO1iycJ9cHSST/5FF3s3ruHfvuV1JXlPEZEwVFe4l3gN91/962MMDE/w2R+/m7p4dX1UIlLdqiuxpsJ9+dMg/98rl3n6yHkef/MO7tzUuuz3ExEJU5WFexc0t8OqDct6m+HxNE9+5Sg72lv40NtKN+tGRCQsiagLKKn8ydRlfsHos8+d5MLVUZ7+mR/UhThEpCJVz5F7NgN9x5fdkjlyOsWffvs0/+rgVu7fprXaRaQyVU+4p3ohPbqsk6ljkxk+8uWj3NbaxEce2lPC4kREwlU9bZmpC3QsfdmB3/vbV+jpH+YLP3WAlobq+WhEpPZUz5F73zGwWPDt1CXoujDIH3yjh39232bevKujxMWJiISresI92Q3rXw91TYv+0clMlo88c5R1LfV88p3luXqTiEiYqijcu5bcb3/qGz0cuzTEr/zYnbQ2l25NGhGRqFRHuI9fgyunlxTup/qu8btfe4V33rWRH73j1tLXJiISgeoI977jwf0ip0Fmss5HnjlKc0Nc10MVkapSHeE+NVNmcQH9p986zXfOXuUX/8k+OlY3lKEwEZFoVEm4d0PDGmgt/rqmZwdG+OxzJ3nr7g5+7Ac2lbE4EZHwVU+4L2LZAXfnY189Sjxm/Oo/vUvXQxWRqlP54e6+6At0PH3kHP9waoCPvWMPt7UtfuqkiMhKV/nhPngOxoeKDvfk0Bi/8tfHeWD7Oh67//YyFyciEo3KD/f8Gu4bFg53d+cTX+1iMpPlN95zNzFdD1VEqlRR4W5mD5nZSTM7ZWZPzvH6z5vZMTM7amZ/a2ZbS1/qPKbCfe+Cu/7l0Ut87XiSD//Ibra1t5S5MBGR6CwY7mYWBz4HPAzsAx4zs9nf0f8usN/d7waeAX6z1IXOK9kNbVuhcc1Ndxu4Ps6nD3Xzhi1t/NQ/2h5ScSIi0SjmyP0AcMrde9x9AvgS8GjhDu7+dXcfyT09DGwubZk3kewu6stLv/SXx7g2Nslnf/xu4mrHiEiVKybcNwHnCp6fz22bz08D/2euF8zscTM7YmZH+vv7i69yPpNjMPDKgidTv3YsyaHvX+SJt+5k1y2rlz+uiMgKV9ITqmb2PmA/8Nm5Xnf3p9x9v7vv7+gowbK6/SfAszcN96GxST7xFy+x59bV/Lu3vG75Y4qIVIBirkhxASj86ufm3LYZzOztwCeAB919vDTlLSB/MvUmbZlff/Y4/dfGeeon9lOfqPzJQSIixSgm7V4AdprZdjOrB94LHCrcwczuAf4AeMTd+0pf5jyS3ZBognVznyD91qnLfPH5c3zgh3bwhi1toZUlIhK1BcPd3dPAE8BzwHHgaXfvNrPPmNkjud0+C6wC/reZfc/MDs3zdqWV7AqmQMbiN7w0MpHmo185yvb2Fn7uR3aFUo6IyEpR1IVC3f1Z4NlZ2z5V8PjtJa6rmKKCcN/9jjlf/q3nXuZcapT/9fhBGutuDH8RkWpWuU3o630wMjDnydQXz1zhj7/Vy08c3MoDO9ZHUJyISLQqN9znWcN9PJ3ho18+ysY1jXz04aVdLFtEpNIV1ZZZkfqOBfez1pT5L393ilN91/mT99/PqobK/eWJiCxHBR+5d8PqjdAy3XY5dnGIz//9q7z73k28ZfeGCIsTEYlWBYd714yWTDqT5SNf/j5tzfV86l2zl74REaktlRnumUnoPzkj3J/6Zg9dF4b45UfvoK25PsLiRESiV5nhPnAKMhNT30x9tf86v/O1V3j4zlt5+K6NERcnIhK9ygz3qWUH7iCbdT76zFGa6uL80qPFX2pPRKSaVWi4d0GsDtbv5AvfPs2RM1f41Lv2sWF1Y9SViYisCBUa7t3QsZtzQ2l+87mTPLirg3ffe7NViEVEakvFhrvfso+Pf/UlDPi1d9+FmS7AISKSV3nhPnoFhi5wdGIz33zlMk8+vIdNbU1RVyUisqJUXrgng2+m/v6JRg5sX8e/fCC8a3GLiFSKivt+vie7MKArvZkvvOduYroeqojIDSou3L81fBsvpd/F+97+ANvbW6IuR0RkRaq4cE9vPsiLuzby+R/aEXUpIiIrVsWF+4O7OnhwVwkuri0iUsUq74SqiIgsSOEuIlKFFO4iIlVI4S4iUoUU7iIiVUjhLiJShRTuIiJVSOEuIlKFzN2jGdisHzizxB9vBy6XsJxKp89jJn0e0/RZzFQNn8dWd1/wm5yRhftymNkRd98fdR0rhT6PmfR5TNNnMVMtfR5qy4iIVCGFu4hIFarUcH8q6gJWGH0eM+nzmKbPYqaa+TwqsucuIiI3V6lH7iIichMVF+5m9pCZnTSzU2b2ZNT1RMXMtpjZ183smJl1m9mHoq5pJTCzuJl918z+KupaomZmbWb2jJmdMLPjZvaDUdcUFTP7udzfky4z+6KZNUZdU7lVVLibWRz4HPAwsA94zMz2RVtVZNLAh919H3AQ+Nka/iwKfQg4HnURK8TvAv/X3fcAb6BGPxcz2wT8e2C/u98JxIH3RltV+VVUuAMHgFPu3uPuE8CXgEcjrikS7n7J3b+Te3yN4C/upmiripaZbQbeCfxh1LVEzcxagTcD/x3A3Sfc/Wq0VUUqATSZWQJoBi5GXE/ZVVq4bwLOFTw/T40HGoCZbQPuATqjrSRyvwN8BMhGXcgKsB3oB/4416b6QzOrySvKu/sF4LeAs8AlYNDd/ybaqsqv0sJdZjGzVcCXgf/g7kNR1xMVM3sX0OfuL0ZdywqRAO4FPu/u9wDDQE2eozKztQT/w98O3Aa0mNn7oq2q/Cot3C8AWwqeb85tq0lmVkcQ7H/u7l+Jup6IvQl4xMxOE7TrftjM/ke0JUXqPHDe3fP/m3uGIOxr0duBXnfvd/dJ4CvAGyOuqewqLdxfAHaa2XYzqyc4KXIo4poiYWZG0E897u6/HXU9UXP3j7n7ZnffRvDn4u/cveqPzubj7q8B58xsd27T24BjEZYUpbPAQTNrzv29eRs1cHI5EXUBi+HuaTN7AniO4Iz3H7l7d8RlReVNwE8AL5nZ93LbPu7uz0ZYk6wsHwT+PHcg1AO8P+J6IuHunWb2DPAdgllm36UGvqmqb6iKiFT1znRBAAAANUlEQVShSmvLiIhIERTuIiJVSOEuIlKFFO4iIlVI4S4iUoUU7iIiVUjhLiJShRTuIiJV6P8DlimKsHdGBwEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.02345964987495172\n",
      "Test accuracy: 0.9919\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Conv2D(32, kernel_size=(3, 3),\n",
    "                 activation='softplus',\n",
    "                 input_shape=input_shape))\n",
    "model.add(Conv2D(64, (3, 3), activation='softplus'))\n",
    "#model.add(AveragePooling2D(pool_size=(2, 2)))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "model.add(Conv2D(64, (3, 3), padding = 'same', activation='softplus'))\n",
    "model.add(Conv2D(64, (3, 3), padding = 'same', activation='softplus'))\n",
    "#model.add(AveragePooling2D(pool_size=(2, 2)))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "model.add(Dropout(0.25))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(128, activation='softplus'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "history = model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))\n",
    "plt.figure()\n",
    "plt.plot(history.history['acc'], label = 'acc')\n",
    "plt.plot(history.history['val_acc'], label = 'val_acc')\n",
    "plt.show()\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2352 0.0001 0.23520000000000002\n"
     ]
    }
   ],
   "source": [
    "# probability of input failure\n",
    "p = 0.0001\n",
    "\n",
    "# calculating n\n",
    "n = np.prod(list(map(int, model.inputs[0].shape[1:])))\n",
    "\n",
    "# n, p, np\n",
    "print(n, p, n * p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAADPtJREFUeJzt3WGIHPUZx/HfL9qA2CqJtcdhYpOWQygSbTmlWtGU1JCGQuwLpUFrSiUnWKFCX1TsiwpS0GJb+sbCVUNiTW0LMSaUWpuGoi3EkItYNdEkNiR6R0wqVpqi2EafvtiJvcbb2cvu7M7ePd8PHLc7z87uw3C/+8/szO7fESEA+cypuwEA9SD8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSOrOXL2abywmBLosIT+dxHY38tlfY3mf7Fdt3dvJcAHrL7V7bb/sMSfslXStpXNIuSasjYm/JOoz8QJf1YuS/XNIrEXEwIv4t6VeSVnXwfAB6qJPwXyDptUn3x4tl/8f2iO0x22MdvBaAinX9Db+IGJU0KrHbD/STTkb+CUkLJ91fUCwDMAN0Ev5dkoZsL7Y9V9LXJG2tpi0A3db2bn9EnLB9u6QnJZ0haV1E7KmsMwBd1fapvrZejGN+oOt6cpEPgJmL8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaTanqJbkmwfknRc0nuSTkTEcBVNAVVYtmxZ09rGjRtL173mmmtK6/v27Wurp37SUfgLX4yINyp4HgA9xG4/kFSn4Q9Jf7C92/ZIFQ0B6I1Od/uviogJ25+QtM32yxHx9OQHFP8U+McA9JmORv6ImCh+H5O0WdLlUzxmNCKGeTMQ6C9th9/22bY/dvK2pOWSXqyqMQDd1clu/4CkzbZPPs8vI+L3lXQFoOvaDn9EHJR0SYW9dNXVV19dWj/vvPNK65s3b66yHfTAZZdd1rQ2NjbWw076E6f6gKQIP5AU4QeSIvxAUoQfSIrwA0lV8am+GWHp0qWl9aGhodI6p/r6z5w55WPX4sWLm9YuvPDC0nWL61dmNUZ+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0gqzXn+m2++ubS+Y8eOHnWCqgwODpbW165d27T2yCOPlK778ssvt9XTTMLIDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJpTnP3+qz35h5HnzwwbbXPXDgQIWdzEwkAkiK8ANJEX4gKcIPJEX4gaQIP5AU4QeSanme3/Y6SV+RdCwiLi6WzZf0a0mLJB2SdENE/KN7bba2ZMmS0vrAwECPOkGvnHvuuW2vu23btgo7mZmmM/Kvl7TilGV3StoeEUOSthf3AcwgLcMfEU9LevOUxaskbShub5B0XcV9Aeiydo/5ByLiSHH7dUnsUwMzTMfX9kdE2I5mddsjkkY6fR0A1Wp35D9qe1CSit/Hmj0wIkYjYjgihtt8LQBd0G74t0paU9xeI2lLNe0A6JWW4bf9qKQdki6yPW77Fkn3SrrW9gFJXyruA5hBWh7zR8TqJqVlFffSkZUrV5bWzzrrrB51gqq0ujZj8eLFbT/3xMRE2+vOFlzhByRF+IGkCD+QFOEHkiL8QFKEH0hq1nx190UXXdTR+nv27KmoE1Tl/vvvL623OhW4f//+prXjx4+31dNswsgPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0nNmvP8ndq1a1fdLcxI55xzTml9xYpTv/j5f2666abSdZcvX95WTyfdc889TWtvvfVWR889GzDyA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBSnOcvzJ8/v7bXvuSSS0rrc+aU/49etqz5t6gvWLCgdN25c+eW1m+88cbSeqve3nnnnaa1nTt3lq777rvvltbPPLP8z3f37t2l9ewY+YGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKUdE+QPsdZK+IulYRFxcLLtb0lpJfy8edldE/K7li9nlL9aBBx54oLR+6623ltZbfb771VdfPe2epmvJkiWlddul9RMnTjStvf3226Xr7t27t7Te6lz82NhYaf2pp55qWjt69GjpuuPj46X1efPmldZbXcMwW0VE+R9MYToj/3pJU30jw08i4tLip2XwAfSXluGPiKclvdmDXgD0UCfH/Lfbft72Otvl+18A+k674f+ZpE9LulTSEUk/avZA2yO2x2yXHxwC6Km2wh8RRyPivYh4X9LPJV1e8tjRiBiOiOF2mwRQvbbCb3tw0t2vSnqxmnYA9ErLj/TaflTSUkkftz0u6fuSltq+VFJIOiSp/DwagL7TMvwRsXqKxQ91oZeO3HbbbaX1w4cPl9avvPLKKts5La2uIdiyZUtpvexc/TPPPNNWT70wMjJSWj///PNL6wcPHqyynXS4wg9IivADSRF+ICnCDyRF+IGkCD+QVJqv7r7vvvvqbgGnKPvK8enYtGlTRZ3kxMgPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0mlOc+P2efxxx+vu4UZjZEfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkmr5eX7bCyU9LGlAUkgajYif2p4v6deSFkk6JOmGiPhH91pFNrZL60NDQ6X1HTt2VNnOrDOdkf+EpO9ExGckfV7St2x/RtKdkrZHxJCk7cV9ADNEy/BHxJGIeLa4fVzSS5IukLRK0obiYRskXdetJgFU77SO+W0vkvRZSTslDUTEkaL0uhqHBQBmiGl/h5/tj0raJOmOiPjn5OOxiAjb0WS9EUkjnTYKoFrTGvltf0SN4G+MiMeKxUdtDxb1QUnHplo3IkYjYjgihqtoGEA1WobfjSH+IUkvRcSPJ5W2SlpT3F4jaUv17QHoluns9n9B0tclvWD7uWLZXZLulfQb27dIOizphu60iKwipjyS/MCcOVym0omW4Y+Iv0hqdsK1swnWAdSGf51AUoQfSIrwA0kRfiApwg8kRfiBpJiiGzPWFVdcUVpfv359bxqZoRj5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApzvOjb7X66m50hpEfSIrwA0kRfiApwg8kRfiBpAg/kBThB5LiPD9q88QTT5TWr7/++h51khMjP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8k5VZzoNteKOlhSQOSQtJoRPzU9t2S1kr6e/HQuyLidy2eq/zFAHQsIqb1RQjTCf+gpMGIeNb2xyTtlnSdpBsk/Ssi7p9uU4Qf6L7phr/lFX4RcUTSkeL2cdsvSbqgs/YA1O20jvltL5L0WUk7i0W3237e9jrb85qsM2J7zPZYR50CqFTL3f4PHmh/VNJTkn4QEY/ZHpD0hhrvA9yjxqHBN1s8B7v9QJdVdswvSbY/Ium3kp6MiB9PUV8k6bcRcXGL5yH8QJdNN/wtd/vd+ArVhyS9NDn4xRuBJ31V0oun2ySA+kzn3f6rJP1Z0guS3i8W3yVptaRL1djtPyTp1uLNwbLnYuQHuqzS3f6qEH6g+yrb7QcwOxF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeS6vUU3W9IOjzp/seLZf2oX3vr174kemtXlb19croP7Onn+T/04vZYRAzX1kCJfu2tX/uS6K1ddfXGbj+QFOEHkqo7/KM1v36Zfu2tX/uS6K1dtfRW6zE/gPrUPfIDqEkt4be9wvY+26/YvrOOHpqxfcj2C7afq3uKsWIatGO2X5y0bL7tbbYPFL+nnCatpt7utj1RbLvnbK+sqbeFtv9ke6/tPba/XSyvdduV9FXLduv5br/tMyTtl3StpHFJuyStjoi9PW2kCduHJA1HRO3nhG1fLelfkh4+ORuS7R9KejMi7i3+cc6LiO/2SW936zRnbu5Sb81mlv6Gatx2Vc54XYU6Rv7LJb0SEQcj4t+SfiVpVQ199L2IeFrSm6csXiVpQ3F7gxp/PD3XpLe+EBFHIuLZ4vZxSSdnlq5125X0VYs6wn+BpNcm3R9Xf035HZL+YHu37ZG6m5nCwKSZkV6XNFBnM1NoOXNzL50ys3TfbLt2ZryuGm/4fdhVEfE5SV+W9K1i97YvReOYrZ9O1/xM0qfVmMbtiKQf1dlMMbP0Jkl3RMQ/J9fq3HZT9FXLdqsj/BOSFk66v6BY1hciYqL4fUzSZjUOU/rJ0ZOTpBa/j9Xczwci4mhEvBcR70v6uWrcdsXM0pskbYyIx4rFtW+7qfqqa7vVEf5dkoZsL7Y9V9LXJG2toY8PsX128UaMbJ8tabn6b/bhrZLWFLfXSNpSYy//p19mbm42s7Rq3nZ9N+N1RPT8R9JKNd7x/5uk79XRQ5O+PiXpr8XPnrp7k/SoGruB/1HjvZFbJJ0nabukA5L+KGl+H/X2CzVmc35ejaAN1tTbVWrs0j8v6bniZ2Xd266kr1q2G1f4AUnxhh+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaT+C6a6AWXDOArCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# obtain input\n",
    "x = np.array([x_train[2]])\n",
    "plt.imshow(x[0])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obtaining experiment from a model\n",
    "exp = ModelInputCrashExperiment(model, p = p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(model, x):\n",
    "    \"\"\" Predict classes, order by most probable \"\"\"\n",
    "    preds = model.predict(x)[0]\n",
    "    sort = np.argsort(-preds)[:3]\n",
    "    print(' '.join(['%d: %.20f%%' % (i, 100 * preds[i]) for i in sort]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4: 99.99997615814208984375% 9: 0.00002134309937673606% 1: 0.00000298075875093673%\n"
     ]
    }
   ],
   "source": [
    "# crashing prediction\n",
    "predict(exp.model_crashing, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4: 99.99997615814208984375% 9: 0.00002134309937673606% 1: 0.00000298075875093673%\n"
     ]
    }
   ],
   "source": [
    "# correct prediction\n",
    "predict(exp.model_correct, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean': array([ 5.3206518e-16,  7.2949806e-11,  5.5989449e-12,  2.0818543e-15,\n",
       "        -4.9569471e-10,  3.8605739e-14,  6.7472881e-12,  3.3007912e-11,\n",
       "         1.7755918e-13,  4.2561382e-10], dtype=float32),\n",
       " 'std': array([6.0661882e-15, 1.0045154e-09, 6.4685222e-11, 2.4253818e-14,\n",
       "        5.7991980e-09, 4.7914548e-13, 8.5031572e-11, 3.7187320e-10,\n",
       "        2.0881448e-12, 5.0924123e-09], dtype=float32)}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# theoretical error\n",
    "exp.get_bound_b3(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean': array([[ 7.5416097e-16,  1.0058494e-10,  7.1183124e-12,  3.1662365e-15,\n",
       "         -2.5033950e-10,  5.9061724e-14,  9.2576962e-12,  4.1548130e-11,\n",
       "          2.4564606e-13,  5.4762639e-10]], dtype=float32),\n",
       " 'std': array([[8.8094501e-15, 1.3049956e-09, 8.5166041e-11, 3.9030496e-14,\n",
       "         7.4399971e-09, 7.5031772e-13, 1.1716579e-10, 4.6027468e-10,\n",
       "         2.8889861e-12, 5.9992353e-09]], dtype=float32)}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get experimental error\n",
    "experiment_mean_std(exp, x, repetitions = 10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}