非線性堆疊網路-2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: KERAS_BACKEND=tensorflow\n"
     ]
    }
   ],
   "source": [
    "%env KERAS_BACKEND=tensorflow "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "C:\\Users\\USER\\anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "C:\\Users\\USER\\anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "C:\\Users\\USER\\anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "C:\\Users\\USER\\anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "C:\\Users\\USER\\anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "C:\\Users\\USER\\anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "C:\\Users\\USER\\anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "C:\\Users\\USER\\anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "C:\\Users\\USER\\anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "C:\\Users\\USER\\anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "C:\\Users\\USER\\anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "C:\\Users\\USER\\anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "# Keras functions\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Activation, Input\n",
    "from keras.optimizers import SGD\n",
    "\n",
    "# Keras dataset\n",
    "from keras.datasets import mnist\n",
    "\n",
    "# Keras utilis function\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = x_train.reshape(60000, 784)\n",
    "x_test = x_test.reshape(10000, 784)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 784) (10000, 784)\n"
     ]
    }
   ],
   "source": [
    "print(x_train.shape, x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = np_utils.to_categorical(y_train, 10) #one-hot encoding\n",
    "y_test = np_utils.to_categorical(y_test, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**1.自定義的不具可訓練權重之神經網路層**\n",
    "\n",
    "自定義神經網路層 (不具可訓練權重-ex：取平均)\n",
    "\n",
    "具有可訓練重的自定義層牽扯到 TensorFlow 及 Python 類別的撰寫，可參考: https://keras.io/layers/writing-your-own-keras-layers/\n",
    "\n",
    "首先需要引入 Lambda 這個函數，透過 Lambda 函數，可以將 Python 上的 function，包裝成 Keras 上的 layer。\n",
    "\n",
    "此外，需要引進後端所使用的套件 (此處為 TensorFlow)，並使用裡面的運算進行 function 的撰寫。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers.core import Lambda\n",
    "from keras import backend as K #Keras 的「後端引擎」"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "透過 backend 來定義一個簡單的 function，作用是對輸入取平均"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average_function(inputs):\n",
    "    return K.mean(inputs, axis=1) #用Keras Backend對變數(tensor)求平均 , axis=1為平均方向"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'numpy.dtype' object has no attribute 'base_dtype'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-686d4c675c43>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#將一個 numpy array 送進這個函式為錯誤\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0maverage_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-14-2e6d905c940f>\u001b[0m in \u001b[0;36maverage_function\u001b[1;34m(inputs)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0maverage_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mK\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m#對變數(tensor)求平均\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36mmean\u001b[1;34m(x, axis, keepdims)\u001b[0m\n\u001b[0;32m   1808\u001b[0m     \u001b[1;33m{\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mnp_implementation\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1809\u001b[0m     \"\"\"\n\u001b[1;32m-> 1810\u001b[1;33m     \u001b[1;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbase_dtype\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbool\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1811\u001b[0m         \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfloatx\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1812\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreduce_mean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'numpy.dtype' object has no attribute 'base_dtype'"
     ]
    }
   ],
   "source": [
    "#將一個 numpy array 送進這個函式為錯誤\n",
    "average_function(np.array([1,3]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "函式不接受 numpy array 的類型作為輸入，這是因為 TensorFlow 有自定義的類型。\n",
    "\n",
    "**需透過 Lambda ，將上述函式包裝成一個神經網路層。**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "average_layer = Lambda(average_function) #將運算包裝成layer，這邊可指定output shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<keras.layers.core.Lambda object at 0x000001C5F4B06D08>\n"
     ]
    }
   ],
   "source": [
    "print(average_layer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "建構出一個將指定長度的資料取平均的神經網路。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputSize = 4 #指定輸入長度為4維\n",
    "\n",
    "x = Input(shape=(inputSize,))\n",
    "y = average_layer(x)\n",
    "average_model = Model(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 4)                 0         \n",
      "_________________________________________________________________\n",
      "lambda_2 (Lambda)            (None,)                   0         \n",
      "=================================================================\n",
      "Total params: 0\n",
      "Trainable params: 0\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "average_model.summary() #無權重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 4)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array([[1,2,3,4]]).shape #一筆4維的向量\n",
    "#在keras以batch為單位輸入(batch = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\USER\\anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([2.5], dtype=float32)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average_model.predict(np.array([[1,2,3,4]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2.5 , 4.25], dtype=float32)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average_model.predict(np.array([[1,2,3,4],\n",
    "                               [1,2,5,9]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**2.具抽樣功能的神經網路層**\n",
    "\n",
    "輸入為 $(\\mu, s)$ ，$\\mu=(\\mu_1,\\cdots,\\mu_n)$ 和 $s=(s_1,\\cdots,s_n)$ 各自為 $n$ 維向量。\n",
    "\n",
    "神經網路層輸出為服從 $N(\\mu, e^{s}I_n)$ 的 $n$ 維向量，換言之，所建構的神經網路其實是一個抽樣函數。\n",
    "\n",
    "** 由於神經網路的輸入輸出經常沒有限制，為了讓 $s$ 具有變異數的非負特性，考慮 $e^{s}$ 作為變異數(非負數)；換言之，$s$ 為 log-variance。\n",
    "\n",
    "假設想進行抽樣的維度為 sampling_dim，則一個具抽樣函數功能的神經網路可由下述方式建構。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampling_dim = 2\n",
    "\n",
    "def sampling(args):\n",
    "    z_mean, z_log_var = args #輸入的向量變成mean和variance\n",
    "    epsilon = K.random_normal(shape=(sampling_dim,),mean=0., stddev=1) #二維的標準常態分配\n",
    "    return z_mean + K.exp(z_log_var / 2) * epsilon #如下公式，得出服從常態分配的值，除二是開根號意思"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "這裡使用到常態分配的線性特性來定義函數，亦即\n",
    "\n",
    "$$X\\sim N(0, 1)\\Rightarrow \\mu+\\sigma X\\sim N(\\mu, \\sigma^2)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#將運算包裝成layer，這邊希望兩個兩維進去，一個兩維出來\n",
    "sample_layer = Lambda(sampling, output_shape=(sampling_dim,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = Input(shape=(sampling_dim,))\n",
    "s = Input(shape=(sampling_dim,))\n",
    "\n",
    "z = sample_layer([m,s]) #Lambda 可接受 list，比較像函式定義的東西，這邊的z是服從常態分配的值\n",
    "\n",
    "sample_model = Model([m,s] , z) #Model(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 2)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, 2)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)               (None, 2)            0           input_1[0][0]                    \n",
      "                                                                 input_2[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 0\n",
      "Trainable params: 0\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "sample_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "平均為 (7, 5)\n"
     ]
    }
   ],
   "source": [
    "#隨機生成mean var\n",
    "test_mean = np.random.randint(10, size = sampling_dim).reshape(1,2)\n",
    "test_log_var = np.array([[0,0]]) #取epsilon變 1\n",
    "\n",
    "print('平均為 (%d, %d)' %(test_mean[0][0], test_mean[0][1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "透過下面的指令，每次可以抽樣出一服從上述要求常態分配之隨機向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[9.203978 , 5.9510236]], dtype=float32)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_model.predict([test_mean, test_log_var])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "和 Numpy 上的抽樣函數進行比較"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_of_samples = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "np_sample = np.random.multivariate_normal(test_mean[0], np.identity(2),\n",
    "                                          size = num_of_samples) #得到100個點"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 3.30895419,  8.68270861],\n",
       "       [ 5.18679229, 10.84715094],\n",
       "       [ 3.7506866 , 10.10014645],\n",
       "       [ 3.81065712, 10.50917616],\n",
       "       [ 3.07652825,  8.92274666],\n",
       "       [ 4.22418775, 10.76595451],\n",
       "       [ 5.05000311,  7.80787424],\n",
       "       [ 3.0488507 ,  8.87904292],\n",
       "       [ 6.02612576,  8.45934205],\n",
       "       [ 3.60284234,  9.1162625 ],\n",
       "       [ 2.19235968,  8.86562778],\n",
       "       [ 4.35344905,  7.93694942],\n",
       "       [ 4.32922576, 10.16191056],\n",
       "       [ 5.91214204,  8.84661494],\n",
       "       [ 3.45965839,  8.49375681],\n",
       "       [ 4.16915475,  8.17277348],\n",
       "       [ 5.59127695, 10.16256918],\n",
       "       [ 5.0039162 ,  7.6641071 ],\n",
       "       [ 1.06717579,  9.06314757],\n",
       "       [ 3.42496556,  7.83297245],\n",
       "       [ 5.3621822 ,  7.38383523],\n",
       "       [ 3.78987345,  8.55932696],\n",
       "       [ 5.13512357,  7.68409893],\n",
       "       [ 5.12050871,  9.94695127],\n",
       "       [ 4.99509304,  9.35926542],\n",
       "       [ 4.15677958,  8.79711615],\n",
       "       [ 3.51582152,  9.65148389],\n",
       "       [ 2.44624318, 10.83841391],\n",
       "       [ 2.16345173,  7.24780661],\n",
       "       [ 4.32088251,  9.53726251],\n",
       "       [ 3.74767691,  8.41172217],\n",
       "       [ 3.65145651,  9.90714729],\n",
       "       [ 2.17356823,  9.07531709],\n",
       "       [ 3.52874027,  9.3313211 ],\n",
       "       [ 5.48390864,  8.23155031],\n",
       "       [ 5.092101  ,  7.71420684],\n",
       "       [ 4.0345597 , 10.14458937],\n",
       "       [ 4.09992183,  8.62278977],\n",
       "       [ 2.75293822,  8.6698601 ],\n",
       "       [ 4.88487414, 10.77745076],\n",
       "       [ 3.66556381,  9.49769605],\n",
       "       [ 3.02204905,  8.396645  ],\n",
       "       [ 2.64278154,  8.87217188],\n",
       "       [ 5.10769289,  9.70684801],\n",
       "       [ 4.44717768,  9.90801319],\n",
       "       [ 3.575983  ,  8.28743009],\n",
       "       [ 3.27317577,  9.15735274],\n",
       "       [ 2.42826331,  8.06966747],\n",
       "       [ 3.39233183,  8.27670036],\n",
       "       [ 2.74022091,  9.53647021],\n",
       "       [ 3.62041195,  8.61370298],\n",
       "       [ 3.95181361, 10.37941553],\n",
       "       [ 3.68585295,  7.72734282],\n",
       "       [ 4.53692949, 10.24991827],\n",
       "       [ 3.68220242,  8.10524664],\n",
       "       [ 2.98039722,  9.39929436],\n",
       "       [ 2.9328196 ,  8.54873302],\n",
       "       [ 3.52852711,  7.7142727 ],\n",
       "       [ 5.59912786,  8.5974186 ],\n",
       "       [ 3.64762721,  9.15497298],\n",
       "       [ 2.99683111, 10.32307484],\n",
       "       [ 1.07828933,  7.83396799],\n",
       "       [ 4.51461121,  9.720833  ],\n",
       "       [ 4.28872046,  6.36270932],\n",
       "       [ 5.66368277,  9.14059403],\n",
       "       [ 2.55738219,  9.0758516 ],\n",
       "       [ 3.49193833, 10.38515674],\n",
       "       [ 4.17237521,  9.04738254],\n",
       "       [ 4.94166132,  9.82506526],\n",
       "       [ 3.10129525,  9.35423508],\n",
       "       [ 4.5038107 ,  8.29155206],\n",
       "       [ 3.94583124,  8.72616965],\n",
       "       [ 3.58986611,  6.27640018],\n",
       "       [ 4.28775952, 10.57916978],\n",
       "       [ 4.37749456,  7.6817213 ],\n",
       "       [ 2.90539967,  9.36141802],\n",
       "       [ 4.84672151,  8.51746686],\n",
       "       [ 5.59487423, 10.19124289],\n",
       "       [ 3.48279021,  7.99979733],\n",
       "       [ 4.81300105,  9.33059541],\n",
       "       [ 3.03246739,  7.36378913],\n",
       "       [ 4.21355422,  7.7781798 ],\n",
       "       [ 2.70180694, 10.39180754],\n",
       "       [ 4.57469928, 10.3208248 ],\n",
       "       [ 3.04506889,  8.45343454],\n",
       "       [ 4.39823734,  9.19096424],\n",
       "       [ 3.9734357 , 10.29612344],\n",
       "       [ 5.0143362 ,  8.39168969],\n",
       "       [ 2.28747107,  9.31054509],\n",
       "       [ 3.28854056, 10.57120509],\n",
       "       [ 4.34987971,  7.59201362],\n",
       "       [ 3.54796017,  9.09442325],\n",
       "       [ 4.1036825 , 10.87162564],\n",
       "       [ 4.11078818,  5.77804986],\n",
       "       [ 3.48169379,  8.52143022],\n",
       "       [ 3.1424712 ,  8.70326769],\n",
       "       [ 3.61598006,  7.8915949 ],\n",
       "       [ 2.92447258,  9.22372217],\n",
       "       [ 2.90783596,  7.08373006],\n",
       "       [ 4.61278573,  8.25664117]])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "kears_sample = np.zeros((num_of_samples, 2)) #100筆2維向量，以下迴圈輸入服從規定常態分配的向量\n",
    "for i in range(num_of_samples):\n",
    "    kears_sample[i] = sample_model.predict([test_mean, test_log_var])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 5.0580492 ,  8.52884293],\n",
       "       [ 2.05793047,  9.77625179],\n",
       "       [ 5.24484634,  9.74450779],\n",
       "       [ 4.39279413,  7.8901186 ],\n",
       "       [ 5.90641069,  9.00927162],\n",
       "       [ 5.0019393 ,  7.02149963],\n",
       "       [ 3.94315434, 10.76341629],\n",
       "       [ 4.63526964,  9.58420277],\n",
       "       [ 3.33372331,  7.56639004],\n",
       "       [ 4.09860849,  7.75074768],\n",
       "       [ 3.97260571,  8.92345047],\n",
       "       [ 4.47688293, 10.1596508 ],\n",
       "       [ 4.77077961, 10.08753777],\n",
       "       [ 5.68126535,  8.36770153],\n",
       "       [ 3.98388577,  8.53683758],\n",
       "       [ 3.68201184,  8.12078857],\n",
       "       [ 2.95955229,  8.02174568],\n",
       "       [ 4.82679272,  9.70175934],\n",
       "       [ 4.06302261,  6.40324545],\n",
       "       [ 4.04698229, 10.06652641],\n",
       "       [ 3.74326777,  8.68317413],\n",
       "       [ 2.08623028,  9.79720592],\n",
       "       [ 2.36143398,  8.72496414],\n",
       "       [ 5.48997831, 10.2927351 ],\n",
       "       [ 3.61952734,  9.16607475],\n",
       "       [ 4.61282635,  7.96361685],\n",
       "       [ 5.26498795, 11.20587349],\n",
       "       [ 4.76329994, 10.58777142],\n",
       "       [ 3.65003777,  8.20364475],\n",
       "       [ 3.50373936,  8.14400673],\n",
       "       [ 1.90202332,  8.0618124 ],\n",
       "       [ 5.55911732,  9.01103687],\n",
       "       [ 5.12151909,  9.26958656],\n",
       "       [ 4.65118551,  7.19066954],\n",
       "       [ 4.47270775, 10.70073509],\n",
       "       [ 2.28062129,  8.32472038],\n",
       "       [ 3.29439402,  7.06217289],\n",
       "       [ 2.91337585, 10.21520424],\n",
       "       [ 4.09773922,  8.56505013],\n",
       "       [ 4.0959568 ,  9.50438118],\n",
       "       [ 5.05664968,  8.38165569],\n",
       "       [ 3.84446287,  7.57171106],\n",
       "       [ 3.20934725,  9.24782944],\n",
       "       [ 4.05488348, 10.9485321 ],\n",
       "       [ 2.90919447,  9.6538763 ],\n",
       "       [ 4.18239307,  9.51695156],\n",
       "       [ 2.70825624, 11.10813713],\n",
       "       [ 3.285429  ,  8.30624866],\n",
       "       [ 4.86247921, 10.0133791 ],\n",
       "       [ 2.5076766 ,  8.37667274],\n",
       "       [ 4.43796635,  8.91289997],\n",
       "       [ 3.22496128, 10.08578873],\n",
       "       [ 2.98734403,  8.66031647],\n",
       "       [ 4.17636299,  9.27566338],\n",
       "       [ 3.03793144,  9.53468609],\n",
       "       [ 3.77359438,  8.86931801],\n",
       "       [ 3.86273384, 10.55433846],\n",
       "       [ 3.43859243,  8.81011009],\n",
       "       [ 3.482831  ,  7.03241062],\n",
       "       [ 3.63790083,  8.91930294],\n",
       "       [ 5.1655612 ,  9.1711359 ],\n",
       "       [ 3.24396658,  6.54010868],\n",
       "       [ 2.73779106,  8.50058651],\n",
       "       [ 3.31443453,  8.60572624],\n",
       "       [ 3.28880024, 12.29779816],\n",
       "       [ 4.34456015, 10.72271824],\n",
       "       [ 2.173069  ,  8.9726944 ],\n",
       "       [ 2.86291313,  8.47503376],\n",
       "       [ 3.98852634,  8.86810112],\n",
       "       [ 3.40029383,  9.34115791],\n",
       "       [ 2.95956087,  7.32669401],\n",
       "       [ 4.73925495,  8.06812477],\n",
       "       [ 3.42937398, 10.7761507 ],\n",
       "       [ 4.64320564,  8.51984215],\n",
       "       [ 4.06181097, 10.74029922],\n",
       "       [ 3.9108367 ,  9.6131115 ],\n",
       "       [ 3.16812181,  7.89866686],\n",
       "       [ 4.07629061,  5.9757309 ],\n",
       "       [ 3.37763357,  8.52441883],\n",
       "       [ 4.24513197,  7.88468552],\n",
       "       [ 3.49027872,  8.45981598],\n",
       "       [ 4.43804073,  8.47264671],\n",
       "       [ 3.27106833,  8.80968571],\n",
       "       [ 3.51506853,  9.8423872 ],\n",
       "       [ 4.04097033,  9.21962833],\n",
       "       [ 3.90967083,  7.94002676],\n",
       "       [ 2.79570985, 10.21218491],\n",
       "       [ 4.69796228,  8.30880737],\n",
       "       [ 3.44391131,  8.70353794],\n",
       "       [ 3.99850965, 12.13938236],\n",
       "       [ 3.82241607,  8.58848476],\n",
       "       [ 4.85181999, 10.09827805],\n",
       "       [ 5.30077982,  9.05641842],\n",
       "       [ 4.16906977,  8.80944252],\n",
       "       [ 5.23601246,  8.7747488 ],\n",
       "       [ 3.34664369,  8.07845783],\n",
       "       [ 4.5034256 ,  9.34775925],\n",
       "       [ 6.16672564,  8.49372864],\n",
       "       [ 3.98098564,  8.72312737],\n",
       "       [ 4.12637234,  9.16479492]])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kears_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Normal Random Samples using Keras/Numpy')"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2dfZgU5ZXof2dglBkUEVAvg+FDY4j4EVExH0Rh40ajUXfUaNYdBTVIvH5vzKiE3atuJJiMe03U5LpsRFEmJkZxTMSPJBrQEGMEwWhEzSqgzJgIKBJh0AHO/aOqh+6equ7q7qqu6u7ze555pruquvq8b1edOu855z2vqCqGYRhG5VEXtwCGYRhGcZgCNwzDqFBMgRuGYVQopsANwzAqFFPghmEYFYopcMMwjArFFHhCEJHrRGR+3HIAiMhdInJD3HKUExFREfl43HIAiMgHIrJf3HIYyadmFLiIrBaRv4nIwLRt00RkUYxiBUJEJovIDvfG/ruIvCoi58UtV6mIyC4i8p8istZt2yoRuTluueJGVXdT1TfCPm+2kSAiI0TkFRG5RUQk7O8LIM+vROQ4Vy4VkTPS9vV3t40ut1yVRM0ocJf+wOWlnkQcyt13Xaq6GzAI+Ffgv0VkbJllCJsZwJHAUcDuwD8Ay2OVqEYQkVHAU8AvVPUyLXBGn4j0L/H7BwJHAIvdTe8C/yEi/Uo5b61Rawq8DfimiAz22ikinxOR50Tkfff/59L2LRKRWSKyBNgC7Oduu0FEfu9akL8UkaEi0i4im9xzjE47xw9E5C133zIRObrQBqjDIzgX/KFBzu1aOPeJyN2uBf9nETkybf94EXne3fczYEBWv1wgIv8jIu+KyC9EpCltn4rIRSLyF/fz3xaR/UXkGVeW+0RkF5/mTAAeVNUut12rVfXutHNfIyKvu+d9WUROTdt3rogsEZGbRWSjiLzh/n7nuv3wjohMTTv+LhG5XUR+7Z5vsavE+iAiu4rITSLypjtqu11EGtx9w0TkYfc73xWRp70e5iIy2u2b/mnbFonINPf1x10Z3heR9W6/p/fpx9Pk/qGILHTlflZE9k879jhxRmTvi8iP3HNO8+nv1Gf2x1HeP1HVq9K27yEid4jI2yLS6V7b/Tz6+13gOvd3flJENrhtaJe0e0tErnbPkxo1HpsmxrHAElX90H3/GPARcLaPzIvS2+XK87usPgt0HYozol0rIt9y5V4tIi3uvgnub57+u50uIity9Wlc1JoCXwosAr6ZvUNEhgALgVuAocD/BRaKyNC0w84BpuNYi2vcbf/sbh8B7A88A9wJDAFWAtemff454DB330+An4tIhrLMh4jUicgpwDDgfwo49ynAT4HBwC+A29zz7QJ0APe4n/05cHra930BmA2cCQx32/3TLLG+hGNNfQa4CpgDtAAfAw4GzvJpzh+Ab7g33iEifYbxrwNHA3sA1wPzRWR42v5PA3/C+b1+4so1Afg4jiK4TUR2Szu+Bfg2Tt+tANp95Pou8Amc/vw4zm/7f9x9VwJrgb2AfYBvAcXUo/g28CtgT2Bf4NYcx56F0/49cX7zWeA8TID7cUYyQ4FXgc/5nCPFfjjK+79U9d+z9s0DtuG0eTxwHJD+MPg08AawtyuD4FwbTcCBOL/3da5sY4FLgAmqujtwPLA67Vwn4txvKRT4d+BaEanP0wY/CrkO/xfOdTACmArMEZGxqvocsAH4YtqxZ+PcH8lDVWviD+fi+UecH/J9nBtwGrDI3X8O8MeszzwDnOu+XgT8R9b+RcDMtPf/CTya9v5kYEUOmd4DPuW+vg6Y73PcZGAHsBH4ENgOXJGnvdnn/k3avnFAt/v6GKALkLT9vwducF/fAXwvbd9uQA8w2n2vwMS0/cuAq7P65Ps+MvYDLgaWuO3qAqbmaNMK4J/c1+cCf0nbd4gryz5p2zYAh7mv7wJ+mtWO7cDH0trxcRyltBnYP+3YzwKr3Nf/ATwEfDxP/492z9k/63qZ5r6+G0fB7OvxWU2d35X7x2n7TgRecV9PAZ5J2yfAW6nv8DjvdcAm9zraP2vfPu5v0JC27Szgt2n9/WaeNjcDy93XHwfewbnn6j2OXZPW99fhXvvAs8D/xnF3atp11tt3afL8LqvPAl2HOPfTNmBg2v77gH93X18NtLuvh+CMuIfnantcf7VmgaOqLwEPA9dk7Wpip1WdYg3OEzrFWx6n/Fva626P970WoIhcKSIr3eHuRhzLclhA0btUdTCOD/wW4AvpOwOc+69pr7cAA9xhYhPQqe7V6pLeDxn9oqof4CjG9H4J3AfpqOp2Vf2hqk7EGRnMAuaKyIFum6aIyArXXbER5+Gb3qbs70FVc3137+/ntuNdt33p7AU0AsvSvvcxdzs4brj/AX4ljtsm+zoKylU4CveP4ri0zs9xbPZvl2pTU1abFGd0kItfAHOBJ7NcSKOAeuDttHb/F461nSLj+heRvUXkp66bZBMwH/f3UdX/Aa7AUc7vuMc1uZ87BNikql73078BM8ly4wWkkOvwPVXdnPZ+DTuvhfnAye7o7UzgaVV9uwh5IqfmFLjLtcAFZCqhLpyLOJ2RQGfa+6JLN4rjk74a54LY01XG7+PcxIFRx2d4NXCIiDSHcO63gRFZ7ouRaa8z+kWc4NNQMvulZFS1W1V/iDNyGOcql//GGYYPddv0EgX2VxYfS71wb84hOO1LZz3OzX6Qqg52//ZQJ4CMqv5dVa9U1f1wRljfyPLtpkgph8a0bf8rrb1/VdULVLUJ+DrwIyk8jfFtHPdLqk2S/t4PVf0GjhHzpIik7oG3cCzwYWntHqSqB6V/NOtUs91th6rqIBxXQ+/vo6o/UdXP41w/iuOagr7uk3TZfo3zgLwoa9dmfPqySPaUtIw0nGu+y5WhE2f0fSrOyDyZ7hNqVIG71sHPgMvSNj8CfEJE/kWcFKav4rgaHg7pa3fHGbatA/qLyP/BsaYLRlU/whkSpvyypZz7Gfezl7ntPg0nKyTFT4DzROQwEdkV+A7wrKquLkb2dETkCjeg1OB+91S3LcuBgTg3/Tr32PNwLPBSOFFEPu/6/b+N044MK1BVd+A8OG4Wkb3d7x4hIse7r08SJwApOO6I7e4fWedZh/OQO1tE+rkWdnrw8QwRSSnb99y29jlPHhbiPsjd0dTFBFdslwBPAk+IyD6uhfkr4D9FZJAba9lfRCblOMfuwAfARvdB0JraISJjReQL7jWzFeehmGrfl3HuNz9m4oxQ0lkBnCYije6D7msB25mL68VJZT0aOAkn/pPibleGQ4AHQ/iuSKhJBe7yHzhKAgBV3YDzI16J4yK4CjhJVdeH9H2PA48Cr+EM17bi7ZIJylxgpIicXMq53YfBaTg+xfeArwIL0vY/gRNcegDH4tsfJ3AbBt04D6K/4li+FwOnq+obqvqyu+8ZnKHwITi+8lL4Cc7o612cYFeLz3FX41iBf3BdA78BUimbB7jvP3Bl+5GqLvI5zwU4Sm0DcBBObCHFBOBZEfkAx61xuaquKqQx7rV5BvA99zvG4QTqP8z1OfezimP5/xH4jRsQnQLsAryMcy3cjxO49uN64HCc0d5C0q4bYFfgRpzf9a84rphvicgeOAHP3+ODqi5x5UrnZpwslb/hBFv9AtBB+StOG7vcc12oqq+k7X8QZ+TwYJarJVFIpuvTMKoTEbkLWKuq/xa3LFEhTjrjWqBFVX8btzxeiMiZwFdU9cwYZZiMEzTN6W4SkdeBr6vqb8oiWBHUsgVuGBWPiBwvIoNdV8W3cHzQf4hZrFxsxLGmE42InI7j1noybllyUdJsKsMwYuezOK6hlOujWVW74xXJH1X9Vdwy5EOc8hrjgHPcmEhiMReKYRhGhWIuFMMwjAqlrC6UYcOG6ejRo8v5lYZhGBXPsmXL1qvqXtnby6rAR48ezdKlS8v5lYZhGBWPiGTPEgfMhWIYhlGxmAI3DMOoUEyBG4ZhVCiWB24YRlnp6elh7dq1bN26NW5REseAAQPYd999qa8PVhLdFLhhGGVl7dq17L777owePZq+a3jULqrKhg0bWLt2LWPGjAn0GXOhGJVHezuMHg11dc7/9lLrGhnlZOvWrQwdOtSUdxYiwtChQwsamZgFblQW7e0wfTps2eK8X7PGeQ/Q4ldc0Egapry9KbRfzAI3KouZM3cq7xRbtjjbDaPGMAVuVBZvvlnYdsPwYLfddq6u9sgjj3DAAQfwZgVeQ6bAjcpi5MjCthuVT4QxjyeeeIJLL72Uxx57jJEBr6Ft27aF9v2lYgrcqCxmzYLGxsxtjY3O9qRhwdbSScU81qwB1Z0xjxD68umnn+aCCy5g4cKF7L+/s9rdunXrOP3005kwYQITJkxgyRJnEajrrruO6dOnc9xxxzFlyhRWr17N0UcfzeGHH87hhx/O73/vLDD09ttvc8wxx3DYYYdx8MEH8/TTT5csZ06iXvY+/e+II45QwyiZ+fNVR41SFXH+z58ft0R9mT9ftbFR1VE7zl9jYzJlLTMvv/xy8INHjcrsw9TfqFElydC/f3/dc8899YUXXsjYftZZZ+nTTz+tqqpr1qzRT37yk6qqeu211+rhhx+uW7ZsUVXVzZs3a3d3t6qqvvbaa5rSbTfddJPecMMNqqq6bds23bRpU8GyefUPsFQ9dKploRiVR0tL8jNOcgVbky57kogo5lFfX8/nPvc57rjjDn7wgx/0bv/Nb37Dyy+/3Pt+06ZN/P3vfwfglFNOoaGhAXAmI11yySWsWLGCfv368dprrwEwYcIEzj//fHp6emhubuawww4rSc58mAvFMKLAgq3hEFHMo66ujvvuu4/nnnuO73znO73bd+zYwTPPPMOKFStYsWIFnZ2d7L777gAMHNi7Bjo333wz++yzDy+88AJLly7lo48+AuCYY47hqaeeYsSIEZxzzjncfffdJcmZtx35DhCRuSLyjoi8lLatTUReEZE/iciDIjI4UikNo9KwYGs4RBjzaGxs5OGHH6a9vZ077rgDgOOOO47bbrut95gVK1Z4fvb9999n+PDh1NXVcc8997B9+3YA1qxZw957780FF1zA1772NZ5//vmS5cxFEAv8LuBLWdt+DRysqocCrwEzQpbLMCqbSgq2JpmWFpgzB0aNAhHn/5w5obmhhgwZwmOPPcYNN9zAQw89xC233MLSpUs59NBDGTduHLfffrvn5y666CLmzZvHZz7zGV577bVe63zRokUcdthhjB8/ngceeIDLL788FDn9CLQmpoiMBh5W1YM99p0KfEVV8/bokUceqbagg1EztLc7Pu8333Qs71mzzP8NrFy5kgMPPDBuMRKLV/+IyDJVPTL72DCCmOcDP/PbKSLTgelA4DxLw6gKKiHYalQ0JQUxRWQmsA3wTcpU1TmqeqSqHrnXXn2WdDMMwzCKpGgLXESmAicBx2oQP4xhGIYRKkUpcBH5EnA1MElVt+Q73jAMwwifIGmE9wLPAGNFZK2IfA24Ddgd+LWIrBAR71CtYRiGERl5LXBVPctj8x0RyGIYhmEUgM3ENAyj5hARrrzyyt73N910E9ddd118AhWJKXDDMBJNx/JOJt74JGOuWcjEG5+kY3lnyefcddddWbBgAevXrw9BwvgwBW4YRmLpWN7JjAUv0rmxGwU6N3YzY8GLJSvx/v37M336dG6++eY++84991zuv//+3vepxR8WLVrEpEmTOPPMM/nEJz7BNddcQ3t7O0cddRSHHHIIr7/+eu/nL7zwQo4++mg+8YlP8PDDDwNw9NFHZ0zNnzhxIn/6059KaocpcMMwEkvb46/S3bM9Y1t3z3baHn+15HNffPHFtLe38/777wf+zAsvvMAPfvADXnzxRe655x5ee+01/vjHPzJt2jRuvfXW3uNWr17N4sWLWbhwIRdeeCFbt25l2rRp3HXXXQC89tprfPjhhxx66KEltcEUuGEYiaVrY3dB2wth0KBBTJkyhVtuuSXwZyZMmMDw4cPZdddd2X///TnuuOMAOOSQQ1i9enXvcWeeeSZ1dXUccMAB7LfffrzyyiucccYZPPzww/T09DB37lzOPffckttgCtwwjMTSNLihoO2FcsUVV3DHHXewefPm3m39+/dnx44dgLPgTapULDi+8xR1dXW97+vq6jKWWsteXV5EaGxs5Itf/CIPPfQQ9913H//yL/9SsvymwA3DSCytx4+lob5fxraG+n60Hj82lPMPGTKEM888s7ecLMDo0aNZtmwZAA899BA9PT0Fn/fnP/85O3bs4PXXX+eNN95g7FhH3mnTpnHZZZcxYcIEhgwZUrL8psANw0gszeNHMPu0QxgxuAEBRgxuYPZph9A8fkRo33HllVdmZKNccMEFLF68mKOOOopnn302YyGHoIwdO5ZJkyZxwgkncPvttzNgwAAAjjjiCAYNGsR5550XiuyBysmGhZWTNQyj2svJnnvuuZx00kl85Stf6bOvq6uLyZMn88orr1BX520/F1JO1ixwwzCMMnD33Xfz6U9/mlmzZvkq70KxRY0NwzBCJJUqmM2UKVOYMmVKqN9lFrhhGGXHKlB7U2i/mAI3DKOsDBgwgA0bNpgSz0JV2bBhQ2/AMwjmQjEMo6zsu+++rF27lnXr1sUtSuIYMGAA++67b+DjTYEbhlFW6uvrGTNmTNxiVAXmQjEMIxm0t8Po0VBX5/xv911q13AxC9wwjPhpb4fp02GLu0LjmjXOe4CWlvjkSjhmgRuGET8zZ+5U3im2bHG2G76YAjeCYcNbI0refLOw7QZgCtwIQmp4u2YNqO4c3poSN8Ji5MjCthuAKXAjCDa8NaJm1ixobMzc1tjobDd8yavARWSuiLwjIi+lbTtDRP4sIjtEpE+BFaPKsOGtETUtLTBnDowaBSLO/zlzLICZhyAW+F3Al7K2vQScBjwVtkBGArHhrVEOWlpg9WrYscP5b8o7L3kVuKo+BbybtW2lqpa+KJ1RGdjwtnQsCGxEgPnAjfzY8LY0LAhsRETkE3lEZDowHWCkDbkrl5YWU9jFkhUE7jhwEm2TptL1pz1oeutJWo8fG+oKM0btELkCV9U5wBxwVuSJ+vsMbzqWd9L2+Kt0beymaXCDKY1ykhbs7ThwEjNOuJTueqfiXOfGbmYseBGgpn4Pux7DwVwoNUDH8k5mLHiRzo3dKDuVRsfyzrhFqw3SRp5tk6b2Ku8U3T3baXu8dkJKdj2GR5A0wnuBZ4CxIrJWRL4mIqeKyFrgs8BCEXk8akGN4ml7/FW6e7ZnbKtqpZG0gGFaELhr0DDPQ7o2dpdToqLpWN7JxBufZMw1C5l445NFKd2aux4jJK8LRVXP8tn1YMiyGBHhpxwqRWkURNxFkdrbHZ/3m286lvesWTu/9/LLadq0ns499u7zsabBDb2vk+peSFnOKeVbrPunpq7HiDEXSg2QrhyCbK9o4pw1mi/bpLub1sXzaOjZmvGxhvp+tB4/FgjgXohxdBGW5VxT12PEmAKvAVqPH0tDfb+MbelKo6qIc9ZoroeHu6955WJmP3orI95/B9EdjNi8gdmnHdJrweZUkvkeEBEr97As55q6HiPG6oHXAOnKIWnD8tAZOdJRbF7boybgw6N55WKaVy523ojArTt69+VUkrfnGV1E7DpqGtxAp4d8hVrONXU9RoyUc2HRI488UpcuXVq27zNqkGwfODgBxHJMPBo92vvhMWqU899v3+rVvW8n3vikp5IcMbiBJd/6R8fyzkbE/8GVdf5SyPaBg2M5p48gjGgQkWWq2qfulLlQjOoizlmjuUoOBCxH4OVeEBxf+MSL76LjwEl9v3fkyHBdRz6umObxI5h92iGMGNyA4DxUTHnHi1ngRsWRmCwNr4wT8M9CyZWhkkaqfZ0buxEg/Q5t6NnK7Edv3emCSY0uZs4MxwKPcwRj+OJngZsCNyqKuIfxOx8eW2jatJ7WRXf1VaYhKTpfd8rmDSz54bmZD4GwFG8uN1BIrhijcMyFYlQFcU4CyUzxEzoH7cWMEy7d6dYIOV3RN6A5cGjfkqthuY6s9ntFYQrcqCjinATi+fCoH0DbpKk7N4So6ArOlw6jnrbVfq8oTIEbFUWck0B8Hx7p0+NDVHSx5Etb7feKwvLAjUAkJXDYevxYTx94LqUWluy+edCb1jsvGhvp+Ob3aLvxyVD6KZZ86ZTVHiDYasSPBTGNvMQdOPSSJ6hSC1N2z3Nt+9DJCtm8io5vfo8ZfxuUmH4yqgfLQjGKJufkkmu+EPg8cVjxYcmeIlcbwv6uMEnKCMooDj8Fbi4UIy9hBA7DqmRXKGEHPZvHj/CV1++cnRu76VjeGZvCjKvvy0LA3PpqxYKYRl7CCBzGlf5XzqBnrnPGsWBBqnb3FT9bUZ31t22tUVPgRn58syF2W+9f/S5rOnbXxqwiTC5Rp/+VM5PD67tSlFthpues+1Hx9bfjLB2cEMyFYuTFMxtit/U0X32ed/U76FMZr2nTejoH7dXn3FGn/5UzkyN1zit+tsJzfzkVpteIJ5uKr7+dY9JRrfj8TYEbgejj+x09Orf1k7WvddFdzDjxMrr779q7rVw1oHP5raP4rlQdk2xKVZiBlJLrE+766q0g/gPsqqi/7VOBsePzp9F6/wv0bHcSNDo3dtN6/wtAFfj8szAXilEcuaZce+zrXcigBirZ5awoWOQ6koEWAk7zCffmpntQNX3vM+no+qOn9irvFD3blet/+ecyClceTIEbxZFryrXPvubNq1hyzRdYdeOXWXLNF4IrkBiXEStmEd/0sqtARkXBYldgDxQETvMJ+y3d9v2vHpa/7736O2kLRYNv/Zf3tnurtfe29JRZwOixPHCjOHJVv4PwSpLGWN40jElAvrnh9dtZ8u1TAssy5pqFeN2pAqy68cvOm7q6jAUfOg6cRNukqXQNGkbTngOD+YG9+ru+3lGQH320c1uCS8yOvmah777Vqb6qMKwaoREuuarfhbmoQoyZBmGkPvrmoX8kBVmxgdIhs0Y+zSsXs+T281n1s0uDj3i8+runJ1N5Q6KzPQY31Be0vZLJq8BFZK6IvCMiL6VtGyIivxaRv7j/94xWTCOR5Kp+F0ZlPIi1vGkYk4B8Fe+m9QUpwEDpkGEUoiqkX73qhieA6045iPo6ydhWXydcd8pBMUkUHUEs8LuAL2VtuwZ4QlUPAJ5w3xtG+MRY3jSMSUCtx4/t64vu2Urr4nkFKctAy5mFMfIppF+lsFFEuWgeP4K2Mz6V0VdtZ3yq8oO2HgTygYvIaOBhVT3Yff8qMFlV3xaR4cAiVc2bk2Q+cKNgKtwHDtBxzFdoO+jLji9603paF89zVvFJ4io3fj7wHp8AYMA21EpedlSEXQtlH1V9G8BV4nvn+OLpwHSAkVYU3iiUMpU3zaVgSlU8zV8/lWavh1CINbZDU5B+/X322d7HBxhFVHUtlpgp1gLfqKqD0/a/p6p5/eBmgRtJpCzlciMsulQW+UtYKzPJVRorhbCzUP7muk5w/79TinCGETk58pjLUmgrrKCuB2WR3yNA2vGpLzLxnFvy5sjHuQxetVOsAv8FkFoIcCrwUDjiGEAyJ01UMnmq1lW6gimL/FkB0o6jT2fGCZfR2dPPf2aoS5zL4FU7QdII7wWeAcaKyFoR+RpwI/BFEfkL8EX3vVEq7e0wbJjjb6zhEpmhkyeXvNIVTNnkTxtFtJ14Ed2amarnZ/XHsrZnjZBXgavqWao6XFXrVXVfVb1DVTeo6rGqeoD7/91yCFvVpKzEDRv67kvwpImKIE8ueaUrGC/56+uELR9tK6gEQCEUYvUHSoG0UWdR2EzMpOBlJaZThokrVUueXPJACibBZMs/uKEexKn9kc+9EQgP5VqI1Z83Q8YWZigaq4WSFLLqWPQhiTnDuUjSUlft7XS0zaPts2ftzMV+5l6aW6cmspZHqYSa9eGTh9/x3TsDLeAcKEOmhAyXWsFqoSSdXDnyIecMR07CLKqOcZOdgNsee6NSR+ceezPjhMvoGDc5FnmiJtSgpk/8oPmmqwKNWgJlyMRYLqHSMQWeFLzqWAAMHZrYqm++JGypq7bHX+0bcFMJN80uQT7cUIOaOZRr8/gRecsDB3qYxFguodIxBZ4UvOpYzJ8P69dXlvKGxFlUkafZJWzEkTcoW8jDpkTlGuhhEkYRrhrFFHiSiHCyR1gEWuAgyE1fRos18jS7Mow4CllYImdQttCHTYnKNVCGT5jlh2sMC2IagQk8ZTtfAaoyF6gKKnfR9UT8AtAizsM4G68AL/gGfUOdKl9MwLDEgLQVsiodvyCmKXAjMAVlN+S66WPIOsinREpSkoW0p4gVb0LNKin0YWMkAlPgRskEWtYrCAlUIiUpyUJGFH7K3gv3ARBav+f6fkvZSzSWRmiUTCm+5Awf7sV30XHgpL4HxZh1UFKgsxAfbiGBXPfYUH34FjCsKoqtB27UIK3Hj/V0M3hNOU93WQxurOeDrdvo2eHYkZ0DhzLjhEsBZ91GcCrbtZ18CV3XLIzFT9o0uMHTAg+sJFNrgeZj5MjgFrj7QCuk3wPJCZFMsjJfd/kxC9wITNAp5yl/cufGbhRnSndKeaforh9A27HnF1zZLid5MltyZXKUrR6KlwVcXw+77JK5Lc0qLniqf74MnwKynYJmv2T/5iVP3zcCYT7wKiBplo+fPzmblA83lCBdHj90kCBl2fqxwCyUgs8dUoZPIYFdW7QhWswHXqUk0fIJOkEm5Z4IZaJNnlzsIFO6g8wsDIUo8/1DzEkvZKGIvL9hgmaqVhOmwCucsqzGUiBB/Mbp7omcQbqgN36e2Z+JXrQhzJmcIc6CLaTP8v6GCZqpWk2YAq9wkqiYWo8fS32d+O7P9uH6+p93Wx/8xs8z+zP2RRtyPYiKtJo9/dMh1hUppM9yxhASVhunmjAFXuHErpg8aB4/gt0GeCc4pXyi6e4J3yDdTVf1ufE7Rk1g4h+29w2q5UmPy6lgoh7e+1mgF12UOy88h9Xs6zr75vdCSxMsJLCbM9Dq1441a8ydUiIWxKxwyrIieRGEMvkka8JPx4GTmHHCpXTXD+jdltHWPFO+PYOULy+C6dPpGDWBtqfEq9IAABlfSURBVElTnXrhf99A67hGmi85s7jGZ+OnpEWKrgGfM2j4sbdDSxMMJbCbb/JShGUU0klasL8QbCZmFZPECzOUrISsG3/ihXPp3GPvks6Z3Vf/8MdHeXjf8WxsGOQoVJeGbR8yu+XT4fRjvsU6vMij1EKdnRk1Xpkx2UQ8EzSphk5Q/BR44ifyJFE5xUGufmgeP6KkPomij0OZfDJrVsaN3zVomOdhQf392Tdx58Zu5h8wKUNxp+juvyttj78azrVWyOQdcJRZHqu55IlH5SR98lAR7qIwyBXsr2R9kmgfeBJT5OIgyn6I6tyhrDOZNUW9act7nocFVVpeN7GX8k4RWiDYyz+f9b0dB05i4oVzGXPVL5l44dy8qwVV3ELMqdTJUaO890dcajiJwf4wKEmBi8jlIvKSiPxZRK4IS6gUSUyRi4Mo+yHKc4eSV52WM916/rElKa1Cb9bQrFmvWikXXtir1FO+fWfJNwn0EK3YhZjz1WIpNeXQR/mXFOxPcA570S4UETkYuAA4CvgIeExEFqrqX8ISrlqfmoUSZT9UUh+nlFOx7h4/t4MXoVuzXrVSJk6EmTNpmzQ1IzALwYb3pbrOYiFfLZZcKYf5gpzZvvaU8gdaj59cnEsvxzmTsOBEKRb4gcAfVHWLqm4DFgOnhiOWQxJT5OKgqX57QdsLOneF9XEpVr2X28GLPRvry2PNuqOLLo/ALCTzIRoKuWailjIRKYfyL3rEkvAc9lIU+EvAMSIyVEQagROBj2UfJCLTRWSpiCxdt25dQV9QcX6+iGhdfDcNPVsztjX0bKV18d2lnzvmPi5kqbBS8bqJz/7MSPe9MmLzBr7/y5tYPne6k15YCgUMu0t+iCZ4iF8wpUxEyqP8i3r4J2x912yKdqGo6koR+S7wa+AD4AVgm8dxc4A54KQRFvIdpQ6Zq4Xm3y2A9eu5/h+n817DIAB27fkQNmwo/dw+fQxOKmBU/d6xvJPrf/ln3tvS07st5ftNlytsPN0O7e3w7aw0t+xhciHLinkNu885B5YsgR/9qM/hJWXsJHyIXzBZmUdA8IlIftk+pdSZj+KcIRJaHriIfAdYq6p9r1AXywMvktGj6Wgc3XcSS5i5ymlEnTPrdf50yl7BLt8qNYVW+Ms1ceeeezw/U3QqZzWusFPsGpxRrLXa3k5H2zzaPnuWM8lr03pan7mX5tapZX1ARlKNUET2dv+PBE4D7i3lfIYPs2bRNvncvoEuN1c5bKLO/vFM50sjp+83qLugELdCvmFyoX5Qv/Open6m47b7aJv7BF3vbaZp8wZad1sf/EGZ8CF+URRbrTGC1e07xk12atXvsTcqdXTusTczTrgsb5pnuSg1D/wBEXkZ+CVwsap6J+oapdHSUvIklkKIOjMl33l8fb9BU8wKTUXL53ctVEnmGl5nfabjtvuYsaofnQOHOgpi4FBmrOpHx233+Z8jyHclZIgfiDB9+CGX6m17/FW6NTNnv1slManMJSlwVT1aVcep6qdU9YmwhDL60jS40Wd7+NkiUWem5DpPTt9vUEu4UIs5X25yoUpy1iz/CUJZn2l7dWvfkVX9ANpezQxa+1Lpa1wmvNRs0tNsEz0T09hJQdkiJVo0UWem+KXzDW6o5/QjRtD2+KveWSlBLeFCLeZ8Q+8ClGTH8k4mvjWcMa2/cGZUpi/e7PGZrsY9PUXy216w7Ekn4Wl6SU+zTXwtFMMhcEZOCFkJ+b6r1NopuTJfsmuVZGSlpGUEdBw4aWf1wC3v0bq8c6cMQ4Z4Z+gMGeIvlN+ixKmA2pYt0K8fbN/uW6skIzgr4vpL3cWbX3kKpvYNfDVteY/OgUP7fK1f2YCCZK8EEu7DD3VB6QiwaoTVRsRZCSVlqOTJLshbwdB9OHWMmpC7rOywYd4KfOhQWL8+eGMLzGrwlf/9d1hy+/mev0HKB57Rlp6tzB6zPbxytkmmArJoklBQz8rJ1gp+pUtFnMBOiRRdJjaAMgxUIrW9nYl/2O5ptfbKUGAf+N6gBSoXX/l1B6u+d0rv9/cpaSvv8du1m+lq3NMZTYwdUBvKG6JJ/XNJguINC1vUuFaIOCuh6KBOAF9nIH9jSwtdA71dIV0b3fMX0Ac5qzHmW0kmaMGkTet7v79jeScz7lue8X0PbNyV1vOPZdV3T2bJrVNqR3lDZD78Wqlkagq82og4K6HQoE7vVPmv3to3qAcZSjJo8LRp87veMqS2F9AHOXPesxT+zpKvv2DiCdfS8cljMrImPOXv2Urr4nmwyy5OPv+CZd5paQuWebapJgg59Q9qp5KpKfBqI+KshNbd1tOw7cOMbX5BnQwrqHcSxKWZSjxNSQYtONT65J3etWGemOtYxTNnOgHDUaPoGDeZiZfMY8ylP2PiW8P7WGA5RxRpD4LMkq9ZbckumFS/HdEdjHj/HWY/eivNKxf3unS6PvK+5fy2R0aBmUoF16yJuT5L0tP/wsIUeDUSgUUDQHs7zVefx+xHbmHE++84SmrTOmbvs2nnmpRpN23bgmV9raD6AbRNmuq88bCKgxQcat68itmP3rpThmxFuWYNzJvHv/3v/+RfT/6mM0kG7zrbOUcUaQ9Dz5Kv6W1JL5h0z2Ws+t4pLLn9fEcmgJ4emDmTpk3eBd38tgeiUGVZYO51we6IBOR2Jyr9L8KHmQUxjeDkCup5FCEac9UvUOlrI4ju4OZn5tE2aQpdPf0KDzAFWGOx48BJ/OvJ30Q9JtSMqN/Okm+f4hwXMKsmb4AyPbCZI4ja8fnTmDHhrL5ZJ8/dS/NT9+dutxfFBAELDM4WHLhOQGZJudbAzBsoDSlIa0FMo3Ry5ex6BCl7g3dZ7NG4KzMmTaOzp19xAaZsN5EHbZOmeipvgK6PpNcKyue2SbkO/Mycpk3r+44kcgRRm79+KrOf+K/M0cMT/0Xz14sspV/MRJgCc68LdkckILe7HCsWBRqZRDxRySzwWqWYim+5LKs33+xjdab8xtn52nUCmz/qW8yq6CqEHnL5Wf/g5mU/en1eazBf1cRey/nrp2b2XT6rq9hqe14UkzYaoQXesbzTKczVuKdTuW/xvJ1upCTkdofY94H6JaS0XrPAjZ0U66PMld3hYXU2r1zM7OfuzbCCTj9ihKfyhhICTB5yNf3du1a6qDpZIQGswVxVE0cMbmD22Z9x3B7ZCiBfIDnMGEUxaaMFZioFzQ7qtUhThbmyg9YnnpizKZETsm8+0Mgk4rReU+DVQKFBkmKHdbkUk49SaP76qRlByd++4h+sK3oFGugjV+u4xj5KR3QHLc8/7FiEAW4gvxtUIP+KLlEFkrMpJm00+3ccOhQaGpxFJzyun6DuCM/UvfRA7yOPFNnIkAjZnREoUBpxWq+5UCqdYoIkUc3WDDA89QsGAnz/q4cFm47v096OcZMzZzh+ci9+u2wVXR9J5nC+sZGO795J2wfDcs7SK3rWabnJ6veOb34vb9syPhvSTMigM1FjI+TrPnCgNAS3jU2lr1aKifjHmCXgpxQHN9Sz4trj8p/AR/aOo09nxqRp3jfTy4v6KLgZfxuU98YrVyZDmOSTuU/WxCM/ovnpB/qeqIhroZhaMGUlguu+XNP1zQderRQT8Y+xhrSfP/W6Uw4KdgKfdrUd9GX/mXdZ7oy2D4Z5Hzv3iQw3VDkyGcIm1wxEz6yJCWf1nR0LRWWM5JyJGtH1VdAEowiu+6IWSg4RKydb6RSz6Gpq+BZWJkQBlLxQtU97uwbt5Xm4lx/bN/jUuGdmcAtobmmJ/KYM04rLFVjL5aPuzRRJUUSQLfO33ULT5ndpffJOmresjqRGefZoI++i2DFe91FhLpRKJ8JqbonEp70Tv3EvnT19F4nIKEXr3rgTL77Lu5phaqifogxD/rDdNLn89l2u5Z1Nr486RYVcPxUTowgBc6FUK5W+Ikuh+LS39bQj/FPdstLHWp+Y611LZfG8vgWrgq5NWSRhF13KlfLnmzWxi1bk9VMr9U5yYQq8GsiTslZwIaKkk97eWbOcQlJHfIzZi3/sFJIiy1+dlT7WvHKxU0tl8wbn2M0bmP3orQB9C1YVssBwEYSthHL57X2V+2lHOP15zz3ORp90wqSRqHonMWEulHIR5uy7AqjETIrABHUf5Usfc88zccptdO6xd5/DRmzewJJbp0TQgPK7AXz97RXoiqvqazuLSNIIReRfgWmAAi8C56mq73LaNavAY7w5EuEnjOrhFTQtLMhx7e2M+dMg3+Jbq757cunyepAYJZSAAlTFUE2r7uQidAUuIiOA3wHjVLVbRO4DHlHVu/w+U7MKPMabI9AyZVES5cMrwMSMjuWdtC1Y5jmZJ1uGiZfe7R3cjNAC75UxbiUU8VJ8RmlEFcTsDzSISH+gEegq8XzVSYzV2WL3E0ZZjS1PnYnevOeefpm1OY4+3fMB0jp2gHdwc2xmHXBPSqj5XEguccnxDD85I67ZYURD0QpcVTuBm4A3gbeB91X1V2EJVlXEeHMELUQUGVE+vPJMzPDNez7xIk/rv/mSM5k9ZrsT3NQdTnAzyOrwZVrAoOR1HnPJGePkLqN4ilbgIrIn8E/AGKAJGCgiZ3scN11ElorI0nXrSlh1pJKJ8eaIfTZhsQ+vIBZtnhTKIBke2RYtEyey5NYphS0wHMUow6P9vimHc35VehGzWktHrRJK8YGfAXxJVb/mvp8CfEZVL/L7TM36wCG2LJTYKcYHHpLfPF8AN7QAYtj+Y5/2j7n0Zyh9F6nonYgTVxEzI3Ki8IG/CXxGRBpFRIBjgZUlnM+XqshjLld50aRRjGVXokWbul46N3b3UXfp7qPQJtGE7SLzaX/T5nc9D+9d+ShfH5XRlVcV92wFUIoP/FngfuB5nBTCOmBOSHL1UrLfz4ifQh9eJfjN068XcPJbU0o8230U2iSaEF1kHcs7mXjCtc5M0AvnZhSaan3yTv9iUSkSUMTM7tnyUVIWiqpeq6qfVNWDVfUcVf0wLMFShD3V2KgASrAUva4XZafbJN01ElqGTqGjDB//fq/iS58JmraiTfPmVTvjGarOepqP3ppZiCpfEbMy+Lntni0fia9GaPUOahCPFe6DWoq5rpfsfOt/+ORePLCsM0PZ1PcTNn+4jTHXLCwsJ7ulJZgizPZvp1U+bHtruH+1wDXPwaxZNI8fkTZz8pLC+yionCXg9xt0ur9BNU60iYvET6VPxExCo/wUGfTNtWDEh9t29AlYnn7ECH77yjq6NnYzuLGeD7Zuo2eHZhwTatZOjkldY/75hz7VApVVh76f0f6dE5TqaNq0jtY/L+y7uHJM+P0G4LizUiOiap01GQUVW40w9jxmIx6KDPr6XS8ieA7rf/vKut5JNI279M9Q3qljQh365/Dv+7p09mzso7x3TlASx9UyaRod4yaHJ2cR5Aoep0j1rvnFwyHxCjz2PGajovC7XjZu6fE8Pn24XxZ3XQ7/flBjJYk+Zq/gcT7ilrkaSLwPHNjp9zOMAHhdL22Pv+o5rE+3epsGN+Q9pmRy+PeDrFbUsbzT1z0RZ1zI66ESBItllUbiLXDD8KWA+iNBrNuC3HXF1j7JkwmSqy5Kysr1I8462MUq4lqq3R0FFWGBG0YfcmRzeNY5CWDdBl6vs8Dv7kORmSC5rNy440J+o5fBDfUM3LV/r1883bUSt8zVQOKzUAzDkzjrV8f03X6lgQG+/9XDYnUzBilLkIiyuRWKXxaKWeBGZRJjid64vtvPyh0xuCF2RRh0hBO3nNWGKXCjMhk50tsKLkf96pi+u/X4sZ5WblLcEKagy48FMY3KJM761TF9d62m1FphLH/MAjfKTii+0FQQMI4SvTF+d61Zudm+9dQEIKCm+sEPC2Ia0ZM2Lb7j86cxY+K5dOvOuXrVupK4UTpWSsOhYqfSGxVO1jJebQd9OUN5g83IM/yxYna5MRdKhVCxKVhZixN0DRrmeZjdkIYXZZkdW8GYBV4BVHSB/KzUut7VY7KwG9LwworZ5cYUeAWQxOJFgclKrWtdPI+Gnq0Z2+yGjIFiSwGUmVrNvAmKuVAqgIr2A2YVb2peuRh22YW2ky+hq6dfZbmDqoVSSwGUmVrLvCkEU+AVQEX7AT1S7ppbp9Lcckq8ctUyuRaNTqACN/wxBV4BJH0GnheZQdfhtD64xKyopBBnGQIjVMwHXgFUmh8w8qBrhfhvE0sJi0YbycIUeIWQq0500og06JqVV97rv/VT4qbs+xJVKQDr67JTtAIXkbEisiLtb5OIXBGmcEZlEmnQNZf/NptClX2tkGdRiaKwvo6FUKbSi0g/oBP4tKp6lGlzsKn0tUGk05/r6hwFkY2IswByOnHWDK81rK8jJeqp9McCr+dS3kbtEOnki0L8txasKx/W17EQlgL/Z+Berx0iMl1ElorI0nXr1oX0dUaSiTToWoj/1oJ15cP6Oh5UtaQ/YBdgPbBPvmOPOOIINYySmT9fddQoVRHn//z5/sc1Nqo6Thfnr7HR/3ijeKyvIwVYqh46NQwL/ATgeVX9WwjnMoz8tLQ4ftUdO5z/fsG3KIJ1hjfW17FQchBTRH4KPK6qd+Y71oKYhmEYhRNJEFNEGoEvAgtKOY9hGIZROCVNpVfVLcDQkGQxDMMwCsBmYhqGYVQopsANwzAqFFPghmEYFYopcMMwjArFFLhhGEaFYgrcqAysVKlh9MFW5DGST4Wt4WgY5cIscCP5FFID3DBqCFPgRvKxUqWG4YkpcCP5VEupUvPjGyFjPnAj+cyalekDh3DWcCwnRfrxO5Z30vb4q3Rt7KZpcAOtx49N9HqoRnkxC9xIPtVQqrQIP37H8k5mLHiRzo3dKNC5sZsZC16kY3lntLIaFUMoa2IGxcrJGjVLIWt5ukS6tqhRUfiVkzUXilGzlNU9MXKk96K/Ofz4XR7KO9d2o/YwF4pRk5TdPVHIWp4uTYMbCtpu1B6mwI2apO3xV+nu2Z6xrbtnO22PvxrNFxbhx289fiwN9f0ytjXU96P1+LHRyGhUHOZCMWqSWNwTLS0FBV5T7hzLQjH8MAVu1CRNgxs8A4RJc080jx9hCtvwxVwoRk1i7gmjGjAL3KhJzD1hVAOmwI2axdwTRqVTkgtFRAaLyP0i8oqIrBSRz4YlmGEYhpGbUn3gPwAeU9VPAp8CVpYukmFUMVbQygiRol0oIjIIOAY4F0BVPwI+Ckcsw6hCbGEKI2RKscD3A9YBd4rIchH5sYgMzD5IRKaLyFIRWbpu3boSvs4wKhxbmMIImVIUeH/gcOD/qep4YDNwTfZBqjpHVY9U1SP32muvEr7OMCocW5jCCJlSFPhaYK2qPuu+vx9HoRuG4UW1LExhJIaiFbiq/hV4S0RSMx+OBV4ORSrDqEaKKGhlGLkoNQ/8UqBdRHYB3gDOK10kw6hSUoHKmTPpGDiGti+cR9fAITS91Ujr8k7LSTcKpiQFrqorgD5Fxg3D8KGlhY5xk5mx4MXeaoipUraAKXGjIKwWimGUmbKXsjWqFlPghlFmbKUdIyxMgRtGmbGVdoywMAVuGGXGStkaYWHVCA2jzFgpWyMsTIEbRgxYKVsjDMyFYhiGUaGYAjcMw6hQTIEbhmFUKKbADcMwKhRT4IZhGBWKqGr5vkxkHbAm4OHDgPURipM0rL3VSy21Fay9UTBKVfssqFBWBV4IIrJUVWumUJa1t3qppbaCtbecmAvFMAyjQjEFbhiGUaEkWYHPiVuAMmPtrV5qqa1g7S0bifWBG4ZhGLlJsgVuGIZh5MAUuGEYRoWSOAUuInNF5B0ReSluWaJGRD4mIr8VkZUi8mcRuTxumaJERAaIyB9F5AW3vdfHLVM5EJF+IrJcRB6OW5aoEZHVIvKiiKwQkaVxyxMlIjJYRO4XkVfce/izZZchaT5wETkG+AC4W1UPjlueKBGR4cBwVX1eRHYHlgHNqvpyzKJFgogIMFBVPxCReuB3wOWq+oeYRYsUEfkGzuLfg1T1pLjliRIRWQ0cqapVP5FHROYBT6vqj0VkF6BRVTeWU4bEWeCq+hTwbtxylANVfVtVn3df/x1YCVRtkWh1+MB9W+/+JcuCCBkR2Rf4MvDjuGUxwkNEBgHHAHcAqOpH5VbekEAFXquIyGhgPPBsvJJEi+tOWAG8A/xaVau6vcD3gauAHXELUiYU+JWILBOR6XELEyH7AeuAO1332I9FZGC5hTAFngBEZDfgAeAKVd0UtzxRoqrbVfUwYF/gKBGpWjeZiJwEvKOqy+KWpYxMVNXDgROAi12XaDXSHzgc+H+qOh7YDFxTbiFMgceM6wt+AGhX1QVxy1Mu3OHmIuBLMYsSJROBU1y/8E+BL4jI/HhFihZV7XL/vwM8CBwVr0SRsRZYmzaCvB9HoZcVU+Ax4gb17gBWqur/jVueqBGRvURksPu6AfhH4JV4pYoOVZ2hqvuq6mjgn4EnVfXsmMWKDBEZ6Abjcd0JxwFVmU2mqn8F3hKRse6mY4GyJx8kblFjEbkXmAwME5G1wLWqeke8UkXGROAc4EXXLwzwLVV9JEaZomQ4ME9E+uEYD/epatWn1tUQ+wAPOnYJ/YGfqOpj8YoUKZcC7W4GyhvAeeUWIHFphIZhGEYwzIViGIZRoZgCNwzDqFBMgRuGYVQopsANwzAqFFPghmEYFYopcMMwjArFFLhhGEaF8v8BFyzVSJxnkc4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(kears_sample[:, 0], kears_sample[:, 1], 'ro')\n",
    "plt.plot(np_sample[:, 0], np_sample[:, 1], 'o')\n",
    "plt.legend(['Keras', 'Numpy'])\n",
    "plt.title('Normal Random Samples using Keras/Numpy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Variational Autoencoder (VAE) 是一個重要的非監督式學習模型，具體應用的場合為特徵抽取/資料壓縮及還原，為影像處理中常見的模型之一。\n",
    "\n",
    "在建立 VAE中，需要的兩個重要技巧:\n",
    "\n",
    "- 分歧-合併\n",
    "- 自定義函數 (抽樣函數)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}