experiment3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.stats as st\n",
    "from sklearn.datasets.samples_generator import make_blobs\n",
    "import scipy\n",
    "import operator\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.applications.inception_v3 import InceptionV3, preprocess_input\n",
    "from keras.applications import Xception, ResNet50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiQAAAFkCAYAAAAQQyCBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3X+UX3V95/HnGxIgCSZpjEkE3MhPgV2EZMQ2m4AcUkvZ\nLmutVjqWsl2OP/JD0VFEZG1RbOlRIYm0zsC6LpLaTA+LC+oWNhXoAQYFZIYfigRckqoQCb+yCcgg\nIXnvH/eOfDOZX0nmzp2ZPB/n3DO5937u5/v55M535jWf+7n3G5mJJElSnfaruwGSJEkGEkmSVDsD\niSRJqp2BRJIk1c5AIkmSamcgkSRJtTOQSJKk2hlIJElS7QwkkiSpdgYSSZJUu8oDSUQsj4gNEdEd\nEXdHxMkDlH13RPwwIjZHxIsRcX9EnNOrzDURsaPXclPV/ZAkSdWZUGXlEXE2cAXwIeBeoAVYGxHH\nZOazfRzyHPBXwDrgFeAs4JqI2JSZ32sodzPw50CU67+upgeSJGkkRJUfrhcRdwP3ZObHyvUAfgFc\nmZlfGmIdncD/zsxLyvVrgGmZ+UcVNVuSJI2wyi7ZRMREoAm4tWdbFunnFmDBEOtYDBwD3N5r12kR\nsSki1kVEa0TMGKZmS5KkGlR5yWYmsD+wqdf2TcBb+jsoIqYCTwIHAq8CyzLztoYiNwPfAjYARwJ/\nA9wUEQuyn+GeiHg9cAbwr8DLe9IZSZL2UQcBbwbWZuZzVb1IpXNI+hHAQNeJXgBOBA4GFgMrI2J9\nZt4BkJnXNZR9OCJ+BDwOnAb8Sz91ngH8w162W5KkfdmfAmuqqrzKQPIssB2Y3Wv7LHYdNfmNcpRj\nfbn6UEQcD3wGuKOf8hsi4lngKPoPJP8K8M1vfpPjjjtuqO0fk1paWli5cmXdzaic/Rxf7Of4s6/0\ndV/o5yOPPMI555wD5e/SqlQWSDJzWzkhdTHwHfjNpNbFwJW7UdV+FJdv+hQRhwGvB345QB0vAxx3\n3HHMnz9/N1567Jk2bdq47yPYz/HGfo4/+0pf95V+liqd8lD1JZsVwLVlMOm57Xcy8A2AiFgNPJGZ\nF5frFwH3UVyCORD4A+AcYEm5fwpwCcUckqcoRkW+CDwGrK24L5IkqSKVBpLMvC4iZgKXUly6eQA4\nIzOfKYscRjFxtccU4Kvl9m6K55H8aWZeX+7fDrwVOBeYDmykCCJ/mZnbquyLJEmqTuWTWjOzFWjt\nZ9/pvdb/AviLAep6Gfj9YW2gJEmqnZ9lM840NzfX3YQRYT/HF/s5/uwrfd1X+jkSKn1S62gREfOB\nzs7Ozn1p8pEkSXutq6uLpqYmgKbM7KrqdRwhkSRJtTOQSJKk2hlIJElS7QwkkiSpdgYSSZJUOwOJ\nJEmqnYFEkiTVzkAiSZJqZyCRJEm1M5BIkqTaGUgkSVLtDCSSJKl2BhJJklQ7A4kkSaqdgUSSJNXO\nQCJJkmpnIJEkSbUzkEiSpNoZSCRJUu0MJJIkqXYGEkmSVDsDiSRJqp2BRJIk1W5C3Q2QJGks6e7u\nZtOmTQDMnj2bSZMm1dyi8cEREkmShqCjo4NTTjmVyZOncPjhh3P44YczefJkjjvu33LzzTfX3bwx\nz0AiSdIgPvnJT3LKKafQ0fEoMBmIcoF16x7hP/yHP2D//SfQ1tZWZzPHtMoDSUQsj4gNEdEdEXdH\nxMkDlH13RPwwIjZHxIsRcX9EnNNHuUsjYmNEvBQR34uIo6rthSRpX/XJT36SFStWAG8AngZ+BbwF\nWAF8p/x6NDt2bGfZsmXMmDGjvsaOYZUGkog4G7gCuASYBzwIrI2Imf0c8hzwV8DvACcA1wDXRMQ7\nG+r8NPAR4MPA2ym+M9ZGxAFV9UOStG/q6OhgxYqVwAHAMxSjIucDDwMfB84qvz4CfBSAzZs3c8AB\n/kraXVWPkLQAV2fm6sxcBywBXgLO66twZt6Rmd/OzEczc0NmXgk8BCxqKPYx4AuZ+d3M/DFwLnAI\n8IeV9kSStM+56KKLgQS2Aa+jGBlZya6/PvcDVgHHALBt2zbe9ra3jVxDx4HKAklETASagFt7tmVm\nArcAC4ZYx2KKs3t7uX44MKdXnVuBe4ZapyRJQ9Hd3c1dd3WUa0kxIP9h+v/VuR+wlJ65JZ2dnVU3\ncVypcoRkJrA/sKnX9k0UoaJPETE1Il6IiFeA7wIfzczbyt1zKL4rdqtOSZJ2V3FrbzZs2QEcOchR\nR+x0zH333Tf8DRun6ngOSbDzGe7tBeBE4GBgMbAyItZn5h17UScALS0tTJs2badtzc3NNDc3D9po\nSdK+LoDHBymznsZfSXfeeeeYunTT3t5Oe3v7Ttu2bNkyIq9dZSB5FtgOzO61fRa7jnD8RnlZZ325\n+lBEHA98BrgDeIriTM/uVccs4P7BGrRy5Urmz58/1PZLkvZhEydO7GPrf6OY1NrXBYYd5f7X/j4+\n+eR+bywdlfr6I72rq4umpqbKX7uySzaZuQ3opBjlACAiolz//m5UtR9wYFnnBopQ0ljnVOC3d7NO\nSZIGtHnz5l5bElhHcb/Gjl77dlDcbbNup61HH310Vc0bd6q+ZLMCuDYiOoF7Kc7iZOAbABGxGngi\nMy8u1y8C7qMYEzsQ+APgHIq7c3qsAj4bEf8X+FfgC8ATwLcr7oskaR9yyCGHNKwFMBGYDvwtxf0Z\nH6SYM7Ie+BrFrb9voHiCxXYApk6dOnINHuMqDSSZeV35zJFLKS6zPACckZnPlEUOA15tOGQK8NVy\nezdF1PzTzLy+oc4vRcRk4GqK74w7gTMz85Uq+yJJ2rfMmDGDN77xMH75y+eBl4FXKB6M9l6K0ZIL\nKILH/hRPnjgO+NZvjn/Pe97r59zshiimbIxvETEf6Ozs7HQOiSRpyFpbW1m+fDnFCMkEiueRQPE8\nkvMobvB8Cvg68NhOx3Z0dLBw4cKRa2xFGuaQNGVmV1Wv46f9SpLUj2XLltHR0VHeebKN4gbQFynC\nx0UUIyW73ujZ1tY2LsLISPLD9SRJGsCaNWtobW3l9a9/A8XD0XoCSE8I6fkanHTSPDo6OliyZElf\nVWkAjpBIkjSIpUuXsnTpUp5//nnWr1/PhAkTmD59OhMnTmTjxo287nWvY+7cuc4Z2QsGEkmShmjG\njBm7fJrvoYceWlNrxhcv2UiSpNoZSCRJUu0MJJIkqXYGEkmSVDsDiSRJqp2BRJIk1c5AIkmSamcg\nkSRJtTOQSJKk2hlIJElS7QwkkiSpdgYSSZJUOwOJJEmqnYFEkiTVzkAiSZJqZyCRJEm1M5BIkqTa\nGUgkSVLtDCSSJKl2BhJJklQ7A4kkSaqdgUSSJNXOQCJJkmpXeSCJiOURsSEiuiPi7og4eYCyH4iI\nOyLi+XL5Xu/yEXFNROzotdxUdT8kSVJ1Kg0kEXE2cAVwCTAPeBBYGxEz+znkHcAa4DTgd4BfAP8c\nEW/sVe5mYDYwp1yah73xkiRpxFQ9QtICXJ2ZqzNzHbAEeAk4r6/CmflnmXlVZj6UmY8BHyjbuLhX\n0V9n5jOZ+XS5bKmyE5IkqVqVBZKImAg0Abf2bMvMBG4BFgyxminAROD5XttPi4hNEbEuIlojYsZw\ntFmSJNWjyhGSmcD+wKZe2zdRXGYZii8CT1KEmB43A+cCpwMXUlzmuSkiYq9aK0mSajOhhtcMIAct\nFHER8D7gHZn5Ss/2zLyuodjDEfEj4HGKeSf/MrxNlSRJI6HKQPIssJ1i8mmjWew6arKTiLiAYvRj\ncWY+PFDZzNwQEc8CRzFIIGlpaWHatGk7bWtubqa52TmxkiS1t7fT3t6+07YtW0ZmmmYU0zoqqjzi\nbuCezPxYuR7Az4ErM/PL/RzzKeBi4Pcy84dDeI3DgJ8B78rM/91PmflAZ2dnJ/Pnz9+zzkiStA/q\n6uqiqakJoCkzu6p6narvslkBfCgizo2IY4GrgMnANwAiYnVEXNZTOCIuBL5AcRfOzyNidrlMKfdP\niYgvRcRvR8TciFgM3Ag8BqytuC+SJKkilc4hyczrymeOXEpx6eYB4IzMfKYschjwasMhSynuqrm+\nV1WfL+vYDryVYlLrdGAjRRD5y8zcVlU/JElStSqf1JqZrUBrP/tO77V++CB1vQz8/vC1TpIkjQZ+\nlo0kSaqdgUSSJNXOQCJJkmpnIJEkSbUzkEiSpNoZSCRJUu0MJJIkqXYGEkmSVDsDiSRJqp2BRJIk\n1c5AIkmSamcgkSRJtTOQSJKk2hlIJElS7QwkkiSpdgYSSZJUOwOJJEmqnYFEkiTVzkAiSZJqZyCR\nJEm1M5BIkqTaGUgkSVLtDCSSJKl2BhJJklQ7A4kkSaqdgUSSJNVuQt0NkKTxrLu7m61bt/LCCy/w\n8MMPM3HiRI444gjmzp3LpEmT6m6eNGoYSCSpAh0dHaxYsYobb7yBzB19ljnhhLfS1tbKwoULR7h1\n0uhT+SWbiFgeERsiojsi7o6Ikwco+4GIuCMini+X7/VVPiIujYiNEfFSWeaoanshSUPX1tbGqaee\nyg033EvmG8utUS6v/ftHP3qIRYsW8alPfaqehkqjSKWBJCLOBq4ALgHmAQ8CayNiZj+HvANYA5wG\n/A7wC+CfI6LnHU1EfBr4CPBh4O3Ar8o6D6ioG5I0ZB0dHSxfvpzME4EngCfLPUcDK4DvlF+P+c0x\nl19+uaFE+7yqR0hagKszc3VmrgOWAC8B5/VVODP/LDOvysyHMvMx4ANlGxc3FPsY8IXM/G5m/hg4\nFzgE+MMqOyJJQ7FixSoypwMPAIdRjIacDzwCfBw4q/z6E+Cjvznu8ssv56677hrx9kqjRWWBJCIm\nAk3ArT3bMjOBW4AFQ6xmCjAReL6s83BgTq86twL37EadklSJ7u5ubrzxBmAzRQg5GTgOWMmuP273\nA1YBb6H4MRdcfvkVI9haaXSpcoRkJrA/sKnX9k0UoWIovkgx3nlLuT4HyL2sU5IqsXXrVoq/u44B\nLgO+DXyQ/n/U7kcxcPwqkNx44w10d3ePSFul0aaO55AERagYuFDERcD7gD/MzFeGo05JqtLEiRPL\nfy0FXgS2A0cOctQRNP742rp1ayVtk0a7Km/7fZbi3Ti71/ZZ7DrCsZOIuAC4EFicmQ837HqKInzM\n7lXHLOD+wRrU0tLCtGnTdtrW3NxMc3PzYIdK0qC2bdtGES6OBKZSDBI/PshR62n8m2rq1KkVtlAa\nWHt7O+3t7Ttt27Jly4i8dhTDixVVHnE3cE9mfqxcD+DnwJWZ+eV+jvkUcDHwe5n5wz72bwS+nJkr\ny/WpFOHk3Mz8n/3UOR/o7OzsZP78+cPQM0naVXd3N1OmHEzmFRQTV99LMZn1R/Q9IL0DOJ4itLzK\nokWncOedd4xYe6Wh6OrqoqmpCaApM7uqep2qL9msAD4UEedGxLHAVcBk4BsAEbE6Ii7rKRwRFwJf\noLgL5+cRMbtcpjTUuQr4bEScFREnAKsp7q37dsV9kaQBTZo0icWLfxdoowgbH6cIJC3leqOe/Y9S\nzCGBiy/+zIi1VRptKn1Sa2ZeVz5z5FKKyywPAGdk5jNlkcPoeScWllJMN7++V1WfL+sgM78UEZOB\nq4HpwJ3AmUOYZyJJlbvoogu55ZbfpQghK4FWYBnF3PwPUswZWU8RWh4rj5rFAQds5cwzz6yhxdLo\nUPmj4zOzleId2de+03utHz7EOj8HfG5v2yZJw23x4sUceuibePLJK4H/Q/F31t8A7cAnKOaKBMVT\nDaB4HuTtrFrV549JaZ/hp/1K0jD7x39cU/7rGYoQchHFPJKTgfcDh1M8ZHoWcDvNzc0sXbq0jqZK\no4aBRJKG2aJFi2hrawP+H8WDpI+lGBm5l2KkZD2QTJnSTWtrK2vWrOm/MmkfYSCRpAosWbKEjo47\nec97FrDffj+l51LNvHnzWLVqFc899xwvvrjVkRGpVPkcEknaVy1cuJCFCxfS3d3N1q1bmTp1KpMm\nTaq7WdKoZCCRpIpNmjTJICINwks2kiSpdgYSSZJUOwOJJEmqnYFEkiTVzkAiSZJqZyCRJEm1M5BI\nkqTaGUgkSVLtDCSSJKl2BhJJklQ7A4kkSaqdgUSSJNXOQCJJkmpnIJEkSbUzkEiSpNoZSCRJUu0M\nJJIkqXYGEkmSVDsDiSRJqp2BRJIk1c5AIkmSamcgkSRJtTOQSJKk2lUeSCJieURsiIjuiLg7Ik4e\noOzxEXF9WX5HRJzfR5lLyn2Ny0+q7YUkSapSpYEkIs4GrgAuAeYBDwJrI2JmP4dMBh4HPg38coCq\nfwzMBuaUy6LharMkSRp5VY+QtABXZ+bqzFwHLAFeAs7rq3Bm3peZn87M64BXBqj31cx8JjOfLpfn\nh7/pkiRppFQWSCJiItAE3NqzLTMTuAVYsJfVHx0RT0bE4xHxzYh4017WJ0mSalTlCMlMYH9gU6/t\nmygus+ypu4E/B86gGHE5HLgjIqbsRZ2SJKlGE2p4zQByTw/OzLUNqz+OiHuBnwHvA64Z6NiWlham\nTZu207bm5maam5v3tDmSJI0b7e3ttLe377Rty5YtI/LaVQaSZ4HtFJNPG81i11GTPZaZWyLiMeCo\nwcquXLmS+fPnD9dLS5I0rvT1R3pXVxdNTU2Vv3Zll2wycxvQCSzu2RYRUa5/f7heJyIOBo5k4Lty\nJEnSKFb1JZsVwLUR0QncS3HXzWTgGwARsRp4IjMvLtcnAsdTXNY5ADg0Ik4EXszMx8syXwa+S3GZ\n5lDg88CrwM5jTJIkacyoNJBk5nXlM0cupbh08wBwRmY+UxY5jCJM9DgEuJ/X5phcUC63A6c3HLMG\neD3wDNAB/E5mPldhVyRJUoUqn9Sama1Aaz/7Tu+1/jMGuYyUmc5AlSRpnPGzbCRJUu0MJJIkqXYG\nEkmSVDsDiSRJqp2BRJIk1c5AIkmSamcgkSRJtTOQSJKk2hlIJElS7QwkkiSpdgYSSZJUOwOJJEmq\nnYFEkiTVzkAiSZJqZyCRJEm1M5BIkqTaGUgkSVLtDCSSJKl2BhJJklQ7A4kkSaqdgUSSJNXOQCJJ\nkmpnIJEkSbUzkEiSpNoZSKRRqru7m02bNtHd3V13UySpcgYSaZTp6Ojgj/7ovRx88OuYM2cOkydP\n4Z3vPIPbbrut7qZJUmUMJNIo0tbWximnnMINN9zJjh1Ty63JLbf8M4sXL+ZNb5rLXXfdVWsbJakK\nBhJplGhtbWXZsmVAAE8Dmxv2BhA88cQTLFq0iKuuuqqWNkpSVSoPJBGxPCI2RER3RNwdEScPUPb4\niLi+LL8jIs7f2zqlsaCtrY3ly5eXawfRE0AKAUwFZgA7AFi6dCltbW0j3UxJqkylgSQizgauAC4B\n5gEPAmsjYmY/h0wGHgc+DfxymOqURrWOjo5yZARgOtANHAOsAL5Tfp0DPFeWKYLKsmXLHCmRNG5U\nPULSAlydmaszcx2wBHgJOK+vwpl5X2Z+OjOvA14Zjjql0e4jHzmfYlRkOvD/gPOBnwAfB84qv/4E\n+Gh5xMHl1+ksW7bMOSWSxoXKAklETASagFt7tmVmArcAC0ZLnVKduru7efDB+4FfA78C3gKsZNe3\n5n7AqnL/i8B7KMLLm1i5ctXINViSKlLlCMlMYH9gU6/tmyjGn0dLnVJtfvazn5X/SuBVigG//t6W\n+5X7obiieQyZB3PDDTf4rBJJY96EGl4zKH76jnidLS0tTJs2badtzc3NNDc3D3NzpKF54oknGtYS\nOHKQI44oy30f+CLwGXbs2MHWrVuZNGlSRa2UtK9ob2+nvb19p21btmwZkdeuMpA8C2wHZvfaPotd\nRzhGpM6VK1cyf/78PXxpafgddNBBDWtBMad7IOt5LX//FrCDiP2YOnXqwIdJ0hD09Ud6V1cXTU1N\nlb92ZZdsMnMb0Aks7tkWEVGuf3+01CnV6fjjj++15Sp6bu3d1Y5y/zSKUPILIHjXu97l6IikMa/q\nu2xWAB+KiHMj4liKn6aTgW8ARMTqiLisp3BETIyIEyPiJOAA4NBy/cih1imNJTNmzGDatN/itVGP\nRyluJOsdSnZQ3G3zKHBgue0fAbjggk+OSFslqUqVziHJzOvK54NcSnGZ5QHgjMx8pixyGMVMvh6H\nAPfz2nyQC8rlduD0IdYpjSmXXfZXDQ9FA7gSWEsxgfUIiss0V1GEkaOBnwKHAj/lggsuYOHChSPc\nYkkaflHcNTu+RcR8oLOzs9M5JBqV3v/+9/eaSNbzlNYs/z2N4lklm+jJ6xdccAFf/vKXR7KZkvZB\nDXNImjKzq6rX8bNspFFgzZo1tLa2MmHCxHJLUlyJ7Pn3VornlCTTp0+no6PDMCJpXDGQSKPE0qVL\n2bbtFZ577jnOO+88Jk1qHL3cwaxZk/nKV77C5s2bvUwjadyp4zkkkgYwY8YMvv71r/P1r3+d559/\nno0bN3LIIYcwY8aMupsmSZUxkEij2IwZMwwikvYJXrKRJEm1M5BIkqTaGUgkSVLtDCSSJKl2BhJJ\nklQ7A4kkSaqdgUSSJNXOQCJJkmpnIJEkSbUzkEiSpNoZSCRJUu0MJJIkqXYGEkmSVDsDiSRJqp2B\nRJIk1c5AIkmSamcgkSRJtTOQSJKk2hlIJElS7QwkkiSpdgYSSZJUOwOJJEmqnYFEkiTVzkAiSZJq\nV3kgiYjlEbEhIroj4u6IOHmQ8n8cEY+U5R+MiDN77b8mInb0Wm6qtheSJKlKlQaSiDgbuAK4BJgH\nPAisjYiZ/ZRfAKwBvgacBNwI3BgRx/cqejMwG5hTLs2VdECSJI2IqkdIWoCrM3N1Zq4DlgAvAef1\nU/5jwM2ZuSIzH83MS4Au4CO9yv06M5/JzKfLZUtlPZAkSZWrLJBExESgCbi1Z1tmJnALsKCfwxaU\n+xut7aP8aRGxKSLWRURrRMwYpmZLkqQaVDlCMhPYH9jUa/smisssfZkzhPI3A+cCpwMXAu8AboqI\n2NsGS5Kkekyo4TUDyD0tn5nXNex7OCJ+BDwOnAb8y0AVtbS0MG3atJ22NTc309zsFBRJktrb22lv\nb99p25YtIzMrospA8iywnWLyaaNZ7DoK0uOp3SxPZm6IiGeBoxgkkKxcuZL58+cPVESSpH1WX3+k\nd3V10dTUVPlrV3bJJjO3AZ3A4p5t5WWVxcD3+znsB43lS+8st/cpIg4DXg/8cm/aK0mS6lP1JZsV\nwLUR0QncS3HXzWTgGwARsRp4IjMvLst/Bbg9Ij4B/BPF7bxNwAfL8lMobiH+FsVoylHAF4HHKCa/\nSpKkMajSQJKZ15XPHLmU4lLMA8AZmflMWeQw4NWG8j+IiGbgr8vlp8C7MvMnZZHtwFspJrVOBzZS\nBJG/LEdkJEnSGFT5pNbMbAVa+9l3eh/bvkUxAtJX+ZeB3x/WBkqSpNr5WTaSJKl2BhJJklQ7A4kk\nSaqdgUSSJNXOQCJJkmpnIJEkSbUzkEiSpNoZSCRJUu0MJJIkqXYGEkmSVDsDiSRJqp2BRJIk1c5A\nIkmSamcgkSRJtTOQSJKk2hlIJElS7QwkkiSpdgYSSZJUOwOJJEmqnYFEkiTVzkAiSZJqZyCRJEm1\nM5BIkqTaGUgkSVLtDCSSJKl2E+pugIbPk08+SWdnJxMmTOCII45g7ty5TJo0qe5mSZI0KAPJGNfd\n3c3SpUu59tpvAtt32X/iiSfx1a/+HQsXLhz5xkmSNESVX7KJiOURsSEiuiPi7og4eZDyfxwRj5Tl\nH4yIM/soc2lEbIyIlyLiexFxVHU9GJ06Ojo48cR5TJ48hWuvvRbYAUS5N37z7wcffIBFixZx1VVX\n1dRSSZIGV2kgiYizgSuAS4B5wIPA2oiY2U/5BcAa4GvAScCNwI0RcXxDmU8DHwE+DLwd+FVZ5wEV\ndmVUaWtr45RTTuGhhx7gtRByNLAC+E759Zidjlm6dCl33XXXiLZTkqShqnqEpAW4OjNXZ+Y6YAnw\nEnBeP+U/BtycmSsy89HMvAToogggjWW+kJnfzcwfA+cChwB/WFkvRpGOjg6WLVtWru1PMTJyPvAI\n8HHgrPLrT4CP7nTsypWrRq6hkiTthsoCSURMBJqAW3u2ZWYCtwAL+jlsQbm/0dqe8hFxBDCnV51b\ngXsGqHNcufjizzasTQaOBVay66ncD1gFvIWeqULf+tb1dHd3j0ArJUnaPVWOkMyk+BN+U6/tmyhC\nRV/mDFJ+NpC7Wee40d3dzZ133l6uLaMYbPow/Z/G/SgGpV6b7Lp169YqmyhJ0h6p4zkkQREqhrP8\n7tY5Jm3atImiq8cAn6UIGkcOctQRNP7XvPrqq1U1T5KkPVblbb/PUvzGnN1r+yx2HeHo8dQg5Z+i\n+I08u1cds4D7B2tQS0sL06ZN22lbc3Mzzc3Ngx06Krz88svlv5YC0ykGoB4f5Kj1NOa1jRs3cuih\nh1bVREnSGNbe3k57e/tO27Zs2TIir11ZIMnMbRHRCSymuPWDiIhy/cp+DvtBH/vfWW4nMzdExFNl\nmYfKOqcCvw18dbA2rVy5kvnz5+9Rf0aDgw46iCJYHAlMopjH+zWKSa19DXbtAHa+3fd1r3tdtY2U\nJI1Zff2R3tXVRVNTU+WvXfUlmxXAhyLi3Ig4luK342TgGwARsToiLmso/xXgzIj4RES8JSI+RzEx\n9u8ayqwCPhsRZ0XECcBq4Ang2xX3pXazZ8+mGO3oGRX5OMXdNS0U4aPRjnL/oxQhZhIQzJ07d2Qa\nK0nSbqj0Sa2ZeV35zJFLKS6zPACckZnPlEUOA15tKP+DiGgG/rpcfgq8KzN/0lDmSxExGbia4rrF\nncCZmflKlX0ZDSZNmsSpp76DO+5ooxgVWQS0UkxwvQX4IMWckfVAG/BYw9HdnHTSPB8lL0kalaK4\nE3d8i4j5QGdnZ+eYvmQDxXNITjnlVIpnjPTc7nsXxcDRDRTTdqLcvn2XY32EvCRpdzRcsmnKzK6q\nXsdP+x1jFi1aRFtbK8U0m+MogshzwELg8LJU0juMtLW1GUYkSaOWH643Bi1ZsoQTTjiBP/mTZp54\n4hMUAaTvO5/f8IY3cMMNNxhGJEmjmiMkY9TChQv5xS9+zq233sLJJ7+d3mGkqelt3HrrrTz99NOG\nEUnSqOcYBz/FAAAJ3UlEQVQIyRh3+umnc++999Dd3V0+OK24G8fJq5KkscRAMk5MmjSJN7/5zXU3\nQ5KkPeIlG0mSVDsDiSRJqp2BRJIk1c5AIkmSamcgkSRJtTOQSJKk2hlIJElS7QwkkiSpdgYSSZJU\nOwOJJEmqnYFEkiTVzkAiSZJqZyCRJEm1M5BIkqTaGUgkSVLtDCSSJKl2BhJJklQ7A4kkSaqdgUSS\nJNXOQCJJkmpnIJEkSbUzkEiSpNoZSMaZ9vb2upswIuzn+GI/x599pa/7Sj9HQmWBJCJ+KyL+ISK2\nRMTmiPjvETFlkGMOjIivRsSzEfFCRFwfEbN6ldnRa9keEe+rqh9jzb7y5rCf44v9HH/2lb7uK/0c\nCVWOkKwBjgMWA38AnApcPcgxq8qy7ynLHwJ8q49y/xmYDcwB3gjcODxNliRJdZhQRaURcSxwBtCU\nmfeX2z4K/FNEXJCZT/VxzFTgPOBPMvP2ctt/AR6JiLdn5r0Nxbdk5jNVtF2SJI28qkZIFgCbe8JI\n6RYggd/u55gmioB0a8+GzHwU+HlZX6OvRsQzEXFPGVokSdIYVskICcWllKcbN2Tm9oh4vtzX3zGv\nZObWXts39TrmL4DbgJeA3wNaI2JKZv7dAO05COCRRx4Zeg/GqC1bttDV1VV3MypnP8cX+zn+7Ct9\n3Rf62fC786BKXygzh7wAfwPsGGDZDhwDfAZ4pI/jnwY+1E/dzUB3H9vvBS4boE2fB342SLvfTzE6\n4+Li4uLi4rJny/t3JzPs7rK7IySXA9cMUmY98BTQ++6Y/YHfohjx6MtTwAERMbXXKMmsAY4BuAf4\nbEQckJmv9FNmLfCnwL8CLw/SfkmS9JqDgDdT/C6tzG4Fksx8DnhusHIR8QNgekTMa5hHshgIigDR\nl07g1bLcDWU9xwD/BvjBAC83j2K+Sn9hpKfdawZrtyRJ6tP3q36BSuaQZOa6iFgLfC0ilgIHAH8L\ntPfcYRMRh1BMYP2zzLwvM7dGxNeBFRGxGXgBuBK4q+cOm4j4jxQjJncDv6aYQ/IZ4EtV9EOSJI2M\nqia1QjFv4+8o7q7ZAVwPfKxh/0SK+SaTG7a1UMxDuR44EPg/wPKG/duAjwArKUZb/i/w8cz879V0\nQZIkjYQoJ31KkiTVxs+ykSRJtTOQSJKk2o2LQDJeP8gvIpZHxIaI6I6IuyPi5EHK/3FEPFKWfzAi\nzuyjzKURsTEiXoqI70XEUdX1YOiGu68RcU0f5++mansxuN3pZ0QcX35fbijbf/7e1jlShrufEXFJ\nH+fzJ9X2YnC72c8PRMQdEfF8uXyvr/Kj8T063P0cJ+/Pd0fED8vfOS9GxP0RcU4f5cb6+Ry0n8N2\nPqt8yMlILcDNQBfwNuDfA48B3xzkmDaK55K8g+LW4e8Dd/YqswP4M+ANFHf3zAIOGKE+nU3xzJRz\ngWMpPpjweWBmP+UXUEz6/QTwFooHxv0aOL6hzKfLOs4C/h3FhxI+PlJ9GuG+XgP8U69zN22M9fNt\nwBeB9wFPAufvbZ1juJ+XAA/1Op8zxtj5/HtgCfBWign9/wPYDLyxocyoe49W1M/x8P48FXhX+TPo\ncOD88ufSO8fZ+RxKP4flfNZ28ofxP/dYiuAwr2HbGRTPNJnTzzFTKX6Bvbth21vKet7esG0H8J9q\n6tfdwFca1gN4Ariwn/L/CHyn17YfAK0N6xuBll7/D93A+2o+h1X09Rrgf9XZr73tZ69jN9D3L+o9\nrnOM9fMSoKvuczic//cUI9RbgHMato2692hF/RxX78+GYzqBz4/n89lPP4flfI6HSzbj7oP8ImJi\n2cbG9iVFv3q3r8eCcn+jtT3lI+IIis8EaqxzK8WD6vqrs3JV9LXBaRGxKSLWRURrRMwYpmbvtj3s\n54jXubcqbtPREfFkRDweEd+MiDftZX17bJj6OYXi8QfPl3Uezih7j1bRzwbj6v0ZEYspRoR6Pq1+\nXJ7P3v1ssNfns8rnkIyU0fZBfsNhJrA/uz4yfxPFSE5f5vRTvqc/sylC2kBl6lBFX6G4jPctir+4\nj6T4HKabImJB+QYcaXvSzzrq3FtVtelu4M+BR4E3Ap8D7oiIf5eZv9qLevfUcPTzixSXqHrC9RxG\n33u0in7COHl/RsRUir4dSDEqvywzbyt3j5vzOUg/YZjO56gNJBHxNxTX3/qTwHEDVVGW2a2XbTwm\nM/+6Yd+DEXEw8CmKB77VYXf7NJTye/L/NBL2qq+ZeV3Dvocj4kcU125PA/5lOBo4TKr4/x+N53Sv\n2pSZjZ+h8eOIuBf4GcW8k8E+X2skDamfEXERRdvfkQN87MXu1DnC9qqf4+j9+QJwInAwxceerIyI\n9Zl5x17UWYe96udwnc/RfMnmcor5If0txzEMH+TXa/tQPsjvsIg4YLd6svuepXhi7exe2wdq31OD\nlH+K4ptud+ocCVX0dReZuaF8rbpmuO9JP+uoc2+NSJsycwvF5PUxdz4j4gLgQopJgQ837BqN79Eq\n+rmLsfr+zML6zHwoM1dSPGX8M+XucXM+B+lnX+X36HyO2kCSmc9l5mODLK9STGacHhHzGg7fnQ/y\nA4bvg/yGQ2ZuK9vY2L4o1/v7gKMfNJYvvbPc3vMN8lSvOqdSzLOp/EOT+lNFX/sSEYcBrwd+uTft\n3VN72M8Rr3NvjVSbytHKIxlj5zMiPgX8V+CMXvPeRuV7tIp+9lN+vLw/96O4rDGuzmcfftPPvuzx\n+dzbWbGjYQFuAu4DTgYWUlxn/vuG/YcAjwBva9jWSnG96zSKST530XDbL/AfgfOA4yl+8C0FXgT+\ncoT69D6K2diNt2Y9B7yh3L8auKyh/ALgFV67FfZzFLd2Nd4Ke2FZx1nACRS3oP2U+m/7Hda+Ukyi\n+xLFG38uxZvtvvJ7YOIY6udEimHSkyiu336xXD9yqHWOo35+meL2w7kUt/Z/j+IvutePoX5eWH6f\nvpviL9SeZUqvMqPqPTrc/RxH78+LgN+luBX2WOCTFHdv/pdxdj4H7Odwns9a/kMq+A+eDnyT4tay\nzcDXgMkN++dSDFOd2rDtQIpPIH6W4vrY/wRmNew/g+LZJluAreW/PzDC/VpG8ayUboq//hsD1W3A\n/+hV/j3AurL8QxR/nfSu83MUt6K9RHFnylF1n7/h7itwEMUHMz5F8YNxPcVzZ2r7Jb0n/Sy/b3eU\n37uNy21DrXO89BNop7g1sZvibrg1wOFjrJ8b+ujjdnr9kTMa36PD2c9x9P78AsUfv7+i+D3SAby3\njzrH+vkcsJ/DeT79cD1JklS7UTuHRJIk7TsMJJIkqXYGEkmSVDsDiSRJqp2BRJIk1c5AIkmSamcg\nkSRJtTOQSJKk2hlIJElS7QwkkiSpdgYSSZJUu/8P53mvwBqSh2MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f9ec514ec88>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def draw(m,s):\n",
    "    X = np.random.normal(m,s, 100*2)\n",
    "    plt.scatter(X[:int(len(X)/2)],X[int(len(X)/2):], s=50)\n",
    "    \n",
    "draw(0,0.001)\n",
    "draw(0.1,0.001)\n",
    "draw(0.2,0.001)\n",
    "draw(0.3,0.001)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.datasets import cifar10\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D, Input, BatchNormalization, GlobalAveragePooling2D\n",
    "import os\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "50000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "noise_dim = 64\n",
    "batch = 64\n",
    "num_classes = 10\n",
    "epochs = 100\n",
    "data_augmentation = True\n",
    "num_predictions = 20\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# Convert class vectors to binary class matrices.\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "label = lambda x: np.array([1,0]) if x == 0 else np.array([0,1])\n",
    "\n",
    "y_noise = np.zeros((len(y_train), noise_dim))\n",
    "for i, yy in enumerate(y_train):\n",
    "    y_noise[i] = np.random.normal(float(yy.argmax()/10),0.0001, noise_dim)\n",
    "\n",
    "def generate():\n",
    "    while True:\n",
    "        step = 0\n",
    "        for b in range(int(len(x_train)/batch)):\n",
    "            itr=0\n",
    "            X = np.zeros((batch,32,32,3))\n",
    "            Y = np.zeros((batch, num_classes))\n",
    "            Yn = np.zeros((batch, noise_dim))\n",
    "            for x_, y_ in zip(x_train[step:step+batch] ,y_train[step:step+batch]):\n",
    "                X[itr] = x_\n",
    "                Y[itr] = y_\n",
    "                Yn[itr] = np.random.normal(float(y_.argmax()/10),0.01, noise_dim)\n",
    "#                 Yn[itr] = yn\n",
    "    \n",
    "                itr+=1\n",
    "                \n",
    "            step+=batch\n",
    "#             yield (X,[Y,Yn])\n",
    "            yield (X,Yn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# inputs = Input(shape = (32,32,3))\n",
    "# x = Conv2D(64, (3, 3), padding=\"same\")(inputs)\n",
    "# x = Activation(\"relu\")(x)\n",
    "# x = BatchNormalization()(x)\n",
    "# x = MaxPooling2D(pool_size=(3, 3))(x)\n",
    "# x = Dropout(0.25)(x)\n",
    " \n",
    "# x = Conv2D(64, (3, 3), padding=\"same\")(x)\n",
    "# x = Activation(\"relu\")(x)\n",
    "# x = BatchNormalization()(x)\n",
    "# x = MaxPooling2D(pool_size=(2, 2))(x)\n",
    "# x = Dropout(0.25)(x)\n",
    "\n",
    "# x = Conv2D(128, (3, 3), padding=\"same\")(x)\n",
    "# x = Activation(\"relu\")(x)\n",
    "# x = BatchNormalization()(x)\n",
    "# x = Dropout(0.25)(x)\n",
    "\n",
    "# x = Conv2D(128, (3, 3), padding=\"same\")(x)\n",
    "# x = Activation(\"relu\")(x)\n",
    "# x = BatchNormalization()(x)\n",
    "# x = Dropout(0.25)(x)\n",
    "\n",
    "# x = Conv2D(128, (3, 3), padding=\"same\")(x)\n",
    "# x = Activation(\"relu\")(x)\n",
    "# x = BatchNormalization()(x)\n",
    "# x = MaxPooling2D(pool_size=(2, 2))(x)\n",
    "# x = Dropout(0.25)(x)\n",
    "\n",
    "# x = Conv2D(256, (3, 3), padding=\"same\")(x)\n",
    "# x = Activation(\"relu\")(x)\n",
    "# x = BatchNormalization()(x)\n",
    "# x = MaxPooling2D(pool_size=(2, 2))(x)\n",
    "# x = Dropout(0.25)(x)\n",
    "\n",
    "# x = GlobalAveragePooling2D()(x)\n",
    "# x = BatchNormalization()(x)\n",
    "# x = Dense(100, activation='relu')(x)\n",
    "\n",
    "# out1 = Dense(10, activation='softmax', name=\"category_output\")(x)\n",
    "# out2 = Dense(noise_dim, activation='linear', name = \"noise\")(x)\n",
    "\n",
    "# #model = Model(inputs=inputs, outputs=[out1,out2])\n",
    "# model = Model(inputs=inputs, outputs=out2)\n",
    "\n",
    "bmodel = Xception(weights='imagenet', include_top=False, input_shape=(None, None, 3))\n",
    "\n",
    "x = GlobalAveragePooling2D()(bmodel.layers[-1].output)\n",
    "x = BatchNormalization()(x)\n",
    "x = Dense(100, activation='linear')(x)\n",
    "out1 = Dense(10, activation='softmax', name=\"category_output\")(x)\n",
    "out2 = Dense(noise_dim, activation='sigmoid', name = \"noise\")(x)\n",
    "model = Model(bmodel.input, [out1,out2])\n",
    "# model = Model(bmodel.input, out2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, None, None, 3 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv1 (Conv2D)           (None, None, None, 3 864         input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv1_bn (BatchNormaliza (None, None, None, 3 128         block1_conv1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv1_act (Activation)   (None, None, None, 3 0           block1_conv1_bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv2 (Conv2D)           (None, None, None, 6 18432       block1_conv1_act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv2_bn (BatchNormaliza (None, None, None, 6 256         block1_conv2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv2_act (Activation)   (None, None, None, 6 0           block1_conv2_bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv1 (SeparableConv2 (None, None, None, 1 8768        block1_conv2_act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv1_bn (BatchNormal (None, None, None, 1 512         block2_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv2_act (Activation (None, None, None, 1 0           block2_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv2 (SeparableConv2 (None, None, None, 1 17536       block2_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv2_bn (BatchNormal (None, None, None, 1 512         block2_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, None, None, 1 8192        block1_conv2_act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2_pool (MaxPooling2D)      (None, None, None, 1 0           block2_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, None, None, 1 512         conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_13 (Add)                    (None, None, None, 1 0           block2_pool[0][0]                \n",
      "                                                                 batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv1_act (Activation (None, None, None, 1 0           add_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv1 (SeparableConv2 (None, None, None, 2 33920       block3_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv1_bn (BatchNormal (None, None, None, 2 1024        block3_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv2_act (Activation (None, None, None, 2 0           block3_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv2 (SeparableConv2 (None, None, None, 2 67840       block3_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv2_bn (BatchNormal (None, None, None, 2 1024        block3_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, None, None, 2 32768       add_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block3_pool (MaxPooling2D)      (None, None, None, 2 0           block3_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, None, None, 2 1024        conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_14 (Add)                    (None, None, None, 2 0           block3_pool[0][0]                \n",
      "                                                                 batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv1_act (Activation (None, None, None, 2 0           add_14[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv1 (SeparableConv2 (None, None, None, 7 188672      block4_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv1_bn (BatchNormal (None, None, None, 7 2912        block4_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv2_act (Activation (None, None, None, 7 0           block4_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv2 (SeparableConv2 (None, None, None, 7 536536      block4_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv2_bn (BatchNormal (None, None, None, 7 2912        block4_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, None, None, 7 186368      add_14[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block4_pool (MaxPooling2D)      (None, None, None, 7 0           block4_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, None, None, 7 2912        conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_15 (Add)                    (None, None, None, 7 0           block4_pool[0][0]                \n",
      "                                                                 batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv1_act (Activation (None, None, None, 7 0           add_15[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv1 (SeparableConv2 (None, None, None, 7 536536      block5_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv1_bn (BatchNormal (None, None, None, 7 2912        block5_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv2_act (Activation (None, None, None, 7 0           block5_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv2 (SeparableConv2 (None, None, None, 7 536536      block5_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv2_bn (BatchNormal (None, None, None, 7 2912        block5_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv3_act (Activation (None, None, None, 7 0           block5_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv3 (SeparableConv2 (None, None, None, 7 536536      block5_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv3_bn (BatchNormal (None, None, None, 7 2912        block5_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_16 (Add)                    (None, None, None, 7 0           block5_sepconv3_bn[0][0]         \n",
      "                                                                 add_15[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv1_act (Activation (None, None, None, 7 0           add_16[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv1 (SeparableConv2 (None, None, None, 7 536536      block6_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv1_bn (BatchNormal (None, None, None, 7 2912        block6_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv2_act (Activation (None, None, None, 7 0           block6_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv2 (SeparableConv2 (None, None, None, 7 536536      block6_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv2_bn (BatchNormal (None, None, None, 7 2912        block6_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv3_act (Activation (None, None, None, 7 0           block6_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv3 (SeparableConv2 (None, None, None, 7 536536      block6_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv3_bn (BatchNormal (None, None, None, 7 2912        block6_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_17 (Add)                    (None, None, None, 7 0           block6_sepconv3_bn[0][0]         \n",
      "                                                                 add_16[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv1_act (Activation (None, None, None, 7 0           add_17[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv1 (SeparableConv2 (None, None, None, 7 536536      block7_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv1_bn (BatchNormal (None, None, None, 7 2912        block7_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv2_act (Activation (None, None, None, 7 0           block7_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv2 (SeparableConv2 (None, None, None, 7 536536      block7_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv2_bn (BatchNormal (None, None, None, 7 2912        block7_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv3_act (Activation (None, None, None, 7 0           block7_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv3 (SeparableConv2 (None, None, None, 7 536536      block7_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv3_bn (BatchNormal (None, None, None, 7 2912        block7_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_18 (Add)                    (None, None, None, 7 0           block7_sepconv3_bn[0][0]         \n",
      "                                                                 add_17[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv1_act (Activation (None, None, None, 7 0           add_18[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv1 (SeparableConv2 (None, None, None, 7 536536      block8_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv1_bn (BatchNormal (None, None, None, 7 2912        block8_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv2_act (Activation (None, None, None, 7 0           block8_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv2 (SeparableConv2 (None, None, None, 7 536536      block8_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv2_bn (BatchNormal (None, None, None, 7 2912        block8_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv3_act (Activation (None, None, None, 7 0           block8_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv3 (SeparableConv2 (None, None, None, 7 536536      block8_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv3_bn (BatchNormal (None, None, None, 7 2912        block8_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_19 (Add)                    (None, None, None, 7 0           block8_sepconv3_bn[0][0]         \n",
      "                                                                 add_18[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv1_act (Activation (None, None, None, 7 0           add_19[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv1 (SeparableConv2 (None, None, None, 7 536536      block9_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv1_bn (BatchNormal (None, None, None, 7 2912        block9_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv2_act (Activation (None, None, None, 7 0           block9_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv2 (SeparableConv2 (None, None, None, 7 536536      block9_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv2_bn (BatchNormal (None, None, None, 7 2912        block9_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv3_act (Activation (None, None, None, 7 0           block9_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv3 (SeparableConv2 (None, None, None, 7 536536      block9_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv3_bn (BatchNormal (None, None, None, 7 2912        block9_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_20 (Add)                    (None, None, None, 7 0           block9_sepconv3_bn[0][0]         \n",
      "                                                                 add_19[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv1_act (Activatio (None, None, None, 7 0           add_20[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv1 (SeparableConv (None, None, None, 7 536536      block10_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv1_bn (BatchNorma (None, None, None, 7 2912        block10_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv2_act (Activatio (None, None, None, 7 0           block10_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv2 (SeparableConv (None, None, None, 7 536536      block10_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv2_bn (BatchNorma (None, None, None, 7 2912        block10_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv3_act (Activatio (None, None, None, 7 0           block10_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv3 (SeparableConv (None, None, None, 7 536536      block10_sepconv3_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv3_bn (BatchNorma (None, None, None, 7 2912        block10_sepconv3[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "add_21 (Add)                    (None, None, None, 7 0           block10_sepconv3_bn[0][0]        \n",
      "                                                                 add_20[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv1_act (Activatio (None, None, None, 7 0           add_21[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv1 (SeparableConv (None, None, None, 7 536536      block11_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv1_bn (BatchNorma (None, None, None, 7 2912        block11_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv2_act (Activatio (None, None, None, 7 0           block11_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv2 (SeparableConv (None, None, None, 7 536536      block11_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv2_bn (BatchNorma (None, None, None, 7 2912        block11_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv3_act (Activatio (None, None, None, 7 0           block11_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv3 (SeparableConv (None, None, None, 7 536536      block11_sepconv3_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv3_bn (BatchNorma (None, None, None, 7 2912        block11_sepconv3[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "add_22 (Add)                    (None, None, None, 7 0           block11_sepconv3_bn[0][0]        \n",
      "                                                                 add_21[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv1_act (Activatio (None, None, None, 7 0           add_22[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv1 (SeparableConv (None, None, None, 7 536536      block12_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv1_bn (BatchNorma (None, None, None, 7 2912        block12_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv2_act (Activatio (None, None, None, 7 0           block12_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv2 (SeparableConv (None, None, None, 7 536536      block12_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv2_bn (BatchNorma (None, None, None, 7 2912        block12_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv3_act (Activatio (None, None, None, 7 0           block12_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv3 (SeparableConv (None, None, None, 7 536536      block12_sepconv3_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv3_bn (BatchNorma (None, None, None, 7 2912        block12_sepconv3[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "add_23 (Add)                    (None, None, None, 7 0           block12_sepconv3_bn[0][0]        \n",
      "                                                                 add_22[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv1_act (Activatio (None, None, None, 7 0           add_23[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv1 (SeparableConv (None, None, None, 7 536536      block13_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv1_bn (BatchNorma (None, None, None, 7 2912        block13_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv2_act (Activatio (None, None, None, 7 0           block13_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv2 (SeparableConv (None, None, None, 1 752024      block13_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv2_bn (BatchNorma (None, None, None, 1 4096        block13_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, None, None, 1 745472      add_23[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block13_pool (MaxPooling2D)     (None, None, None, 1 0           block13_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, None, None, 1 4096        conv2d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_24 (Add)                    (None, None, None, 1 0           block13_pool[0][0]               \n",
      "                                                                 batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv1 (SeparableConv (None, None, None, 1 1582080     add_24[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv1_bn (BatchNorma (None, None, None, 1 6144        block14_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv1_act (Activatio (None, None, None, 1 0           block14_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv2 (SeparableConv (None, None, None, 2 3159552     block14_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv2_bn (BatchNorma (None, None, None, 2 8192        block14_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv2_act (Activatio (None, None, None, 2 0           block14_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_2 (Glo (None, 2048)         0           block14_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 100)          204900      global_average_pooling2d_2[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "noise (Dense)                   (None, 64)           6464        dense_2[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 21,072,844\n",
      "Trainable params: 21,018,316\n",
      "Non-trainable params: 54,528\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "losses = {\n",
    "    \"category_output\": \"binary_crossentropy\",\n",
    "    \"noise\": keras.losses.MAE,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=losses,\n",
    "              optimizer=keras.optimizers.Adam(lr=0.0000652),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit_generator(generate(), \n",
    "                    steps_per_epoch=int(len(x_train)/batch),\n",
    "                    epochs=2000\n",
    "                   )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('exp3.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vec_model=Model(model.input, model.layers[-2].output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "#vecs = np.zeros((len(x_train),noise_dim))\n",
    "vecs = np.zeros((len(x_test),100))\n",
    "for i,x in enumerate(x_test):\n",
    "    #pred = model.predict(np.expand_dims(x,axis=0))\n",
    "    #vecs[i] = pred[1][0]\n",
    "    pred = vec_model.predict(np.expand_dims(x,axis=0))\n",
    "    vecs[i] = pred[0]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def similar(ind):\n",
    "    #pred = model.predict(np.expand_dims(x_train[ind],axis=0))\n",
    "    #vec_query = pred[1][0]\n",
    "    \n",
    "    pred = vec_model.predict(np.expand_dims(x_train[ind],axis=0))\n",
    "    vec_query = pred[0]\n",
    "#     print(vec_query)\n",
    "    dict_={}\n",
    "    dict_cos={}\n",
    "    for i,x in enumerate(vecs):\n",
    "        distance = scipy.spatial.distance.euclidean(x, vec_query)\n",
    "        distance_cos = scipy.spatial.distance.cosine(x, vec_query)\n",
    "        \n",
    "        if distance in dict_:\n",
    "            distance+=(0.0000001/i)\n",
    "        dict_.update({distance:i})\n",
    "        \n",
    "        if distance_cos in dict_cos:\n",
    "            distance_cos+=(0.0000001/i)\n",
    "        dict_cos.update({distance_cos:i})        \n",
    "    \n",
    "    return dict_,dict_cos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ind = np.random.randint(len(y_train))\n",
    "#33078 46993 49426 39277 211 131 457\n",
    "plt.imshow(x_train[ind])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "dict_, dict_cos = similar(ind)\n",
    "sorted_x = sorted(dict_.items(), key=operator.itemgetter(0), reverse=False)\n",
    "sorted_c = sorted(dict_cos.items(), key=operator.itemgetter(0), reverse=False)\n",
    "for i,c in zip(sorted_x[:10],sorted_c[:10]):\n",
    "    plt.figure(figsize=(2,4))\n",
    "    \n",
    "    plt.imshow(np.concatenate((x_test[i[1]],x_test[c[1]]),axis=0))\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "from sklearn.manifold import TSNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "tsne = TSNE().fit_transform(vecs)\n",
    "tx, ty = tsne[:,0], tsne[:,1]\n",
    "tx = (tx-np.min(tx)) / (np.max(tx) - np.min(tx))\n",
    "ty = (ty-np.min(ty)) / (np.max(ty) - np.min(ty))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "width = 4000\n",
    "height = 3000\n",
    "max_dim = 100\n",
    "full_image = Image.new('RGB', (width, height))\n",
    "for idx, x in enumerate(x_test):\n",
    "    tile = Image.fromarray(np.uint8(x * 255))\n",
    "    rs = max(1, tile.width / max_dim, tile.height / max_dim)\n",
    "    tile = tile.resize((int(tile.width / rs),\n",
    "                        int(tile.height / rs)),\n",
    "                       Image.ANTIALIAS)\n",
    "    full_image.paste(tile, (int((width-max_dim) * tx[idx]),\n",
    "                            int((height-max_dim) * ty[idx])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_image.save('full_image_xception_noise_MAE_1.jpg')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}