dragan-lasagne.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['THEANO_FLAGS'] = 'floatX=float32,device=cuda' #,optimizer=fast_compile'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import numpy as np\n",
    "import tarfile\n",
    "\n",
    "# 下載 dataset\n",
    "url = \"https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\"\n",
    "import os\n",
    "import urllib\n",
    "from urllib.request import urlretrieve\n",
    "def reporthook(a,b,c):\n",
    "    print(\"\\rdownloading: %5.1f%%\"%(a*b*100.0/c), end=\"\")\n",
    "tar_gz = \"cifar-10-python.tar.gz\"\n",
    "if not os.path.isfile(tar_gz):\n",
    "        print('Downloading data from %s' % url)\n",
    "        urlretrieve(url, tar_gz, reporthook=reporthook)\n",
    "\n",
    "import pickle\n",
    "train_X=[]\n",
    "train_y=[]\n",
    "tar_gz = \"cifar-10-python.tar.gz\"\n",
    "with tarfile.open(tar_gz) as tarf:\n",
    "    for i in range(1, 6):\n",
    "        dataset = \"cifar-10-batches-py/data_batch_%d\"%i\n",
    "        print(\"load\",dataset)\n",
    "        with tarf.extractfile(dataset) as f:\n",
    "            result = pickle.load(f, encoding='latin1')\n",
    "        train_X.extend( result['data'].reshape(-1,3,32,32)/255*2-1)\n",
    "        train_y.extend(result['labels'])\n",
    "    train_X=np.float32(train_X)\n",
    "    train_y=np.int32(train_y)\n",
    "    dataset = \"cifar-10-batches-py/test_batch\"\n",
    "    print(\"load\",dataset)\n",
    "    with tarf.extractfile(dataset) as f:\n",
    "        result = pickle.load(f, encoding='latin1')\n",
    "        test_X=np.float32(result['data'].reshape(-1,3,32,32)/255*2-1)\n",
    "        test_y=np.int32(result['labels'])\n",
    "train_X = np.concatenate([train_X, test_X])\n",
    "train_X = np.concatenate([train_X[:,:,:,::-1], train_X])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from IPython.display import display\n",
    "def showX(X, rows=1):\n",
    "    assert X.shape[0]%rows == 0\n",
    "    int_X = ( (X+1)/2*255).clip(0,255).astype('uint8')\n",
    "    int_X = np.moveaxis(int_X, 1, 3)\n",
    "    int_X_reshape = int_X.reshape(rows, -1, 32, 32,3).swapaxes(1,2).reshape(rows*32,-1, 3)\n",
    "    display(Image.fromarray(int_X_reshape))\n",
    "# 訓練資料， X 的前 20 筆\n",
    "showX(train_X[:20])\n",
    "print(train_y[:20])\n",
    "name_array = np.array(\"飛機、汽車、鳥、貓、鹿、狗、青蛙、馬、船、卡車\".split('、'))\n",
    "print(name_array[train_y[:20]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "import lasagne\n",
    "from lasagne.layers import DenseLayer, DropoutLayer, ReshapeLayer, InputLayer, FlattenLayer, Upscale2DLayer, LocalResponseNormalization2DLayer\n",
    "floatX = theano.config.floatX\n",
    "from lasagne.layers import MaxPool2DLayer, Conv2DLayer, TransposedConv2DLayer\n",
    "from lasagne.layers import batch_norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "conv_init = lasagne.init.Normal(0.05, 0)\n",
    "gamma_init = lasagne.init.Normal(0.02, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def DCGAN_D(isize, nz, nc, ndf, n_extra_layers=0):\n",
    "    _ = InputLayer(shape=(None, nc, isize, isize))\n",
    "    _ = Conv2DLayer(_, num_filters=ndf, filter_size=4, stride=2, pad=1, b=None, W=conv_init, flip_filters=False,\n",
    "               name = 'initial.conv.{0}-{1}'.format(nc, ndf), \n",
    "                nonlinearity=lasagne.nonlinearities.LeakyRectify(0.2))\n",
    "    csize, cndf = isize // 2, ndf\n",
    "    while csize > 5:\n",
    "        in_feat = cndf\n",
    "        out_feat = cndf*2\n",
    "        _ = Conv2DLayer(_, num_filters=out_feat, filter_size=4, stride=2, pad=1, b=None, W=conv_init, \n",
    "                                   flip_filters=False,\n",
    "               name = 'pyramid.{0}-{1}.conv'.format(in_feat, out_feat), \n",
    "                nonlinearity=lasagne.nonlinearities.LeakyRectify(0.2))\n",
    "        if 0: # change this line to turn on batch_norm\n",
    "            _ = batch_norm(_, epsilon=1e-5)\n",
    "        csize, cndf = csize//2, cndf*2\n",
    "        \n",
    "    _ = Conv2DLayer(_, num_filters=1, filter_size=csize, stride=1, pad=0, b=None, W=conv_init, \n",
    "                    flip_filters=False,\n",
    "               name = 'final.{0}-{1}.conv'.format(cndf, 1), \n",
    "                nonlinearity=None)\n",
    "    _ = FlattenLayer(_)\n",
    "    return _\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def DCGAN_G(isize, nz, nc, ngf, n_extra_layers=0):\n",
    "    cngf= ngf//2\n",
    "    tisize = isize\n",
    "    while tisize > 5:\n",
    "        cngf = cngf * 2\n",
    "        tisize = tisize // 2\n",
    "    _ = InputLayer(shape=(None, nz))\n",
    "    _ = ReshapeLayer(_, (-1, nz, 1,1))\n",
    "    _ = TransposedConv2DLayer(_, num_filters=cngf, filter_size=tisize, stride=1, crop=0, b=None, \n",
    "                              W=conv_init,\n",
    "               name =  'initial.{0}-{1}.convt'.format(nz, cngf))\n",
    "    _ = batch_norm(_, epsilon=1e-5)\n",
    "    csize, cndf = tisize, cngf\n",
    "    \n",
    "    while csize < isize//2:\n",
    "        in_feat = cngf\n",
    "        out_feat = cngf//2\n",
    "        _ = TransposedConv2DLayer(_, num_filters=out_feat, filter_size=4, stride=2, crop=1, b=None, W=conv_init,\n",
    "               name = 'pyramid.{0}-{1}.convt'.format(in_feat, out_feat))\n",
    "        _ = batch_norm(_, epsilon=1e-5)\n",
    "        csize, cngf = csize*2, cngf//2\n",
    "    _ = TransposedConv2DLayer(_, num_filters=nc, filter_size=4, stride=2, crop=1, b=None, W=conv_init,\n",
    "               name = 'final.{0}-{1}.convt'.format(cngf, nc), nonlinearity=lasagne.nonlinearities.tanh)       \n",
    "    return _\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nc = 3\n",
    "nz = 24\n",
    "ngf = 64\n",
    "ndf = 64\n",
    "n_extra_layers = 0\n",
    "Diters = 5\n",
    "\n",
    "imageSize = 32\n",
    "batchSize = 64\n",
    "lrD = 0.0001\n",
    "lrG = 0.0001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "netD = DCGAN_D(imageSize, nz, nc, ndf, n_extra_layers)\n",
    "for l in lasagne.layers.get_all_layers(netD):\n",
    "    print(l.name,  l.output_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "netG = DCGAN_G(imageSize, nz, nc, ngf, n_extra_layers)\n",
    "for l in lasagne.layers.get_all_layers(netG):\n",
    "    print(l.name,  l.output_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "input_var_D = lasagne.layers.get_all_layers(netD)[0].input_var\n",
    "input_var_G = lasagne.layers.get_all_layers(netG)[0].input_var\n",
    "ϵ = T.tensor4()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "no_bn_avg = dict(       batch_norm_update_averages=False,\n",
    "                       batch_norm_use_averages=False)\n",
    "output_D = lasagne.layers.get_output(netD, **no_bn_avg)\n",
    "output_G = lasagne.layers.get_output(netG, **no_bn_avg)\n",
    "\n",
    "output_D_fake = lasagne.layers.get_output(netD, inputs=output_G, **no_bn_avg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mixed_X =  input_var_D + ϵ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "output_D_mixed = lasagne.layers.get_output(netD, inputs=mixed_X, **no_bn_avg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "grad_mixed = T.grad(T.sum(output_D_mixed), mixed_X)\n",
    "norm_grad_mixed = T.sqrt(T.sum(T.square(grad_mixed),axis=[1,2,3]))\n",
    "grad_penalty = T.mean(T.square(norm_grad_mixed -1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "loss_D_real = output_D.mean()\n",
    "loss_D_fake = output_D_fake.mean()\n",
    "loss_D = loss_D_fake - loss_D_real + 10 * grad_penalty\n",
    "loss_G = -loss_D_fake\n",
    "\n",
    "\n",
    "params_netD = lasagne.layers.get_all_params(netD, trainable=True) \n",
    "params_netG = lasagne.layers.get_all_params(netG, trainable=True)\n",
    "#optimize_G = lasagne.updates.rmsprop(loss_G, params_netG, learning_rate=lrG)\n",
    "optimize_G = lasagne.updates.adam(loss_G, params_netG, learning_rate=lrG, beta1=0.0, beta2=0.9)\n",
    "#optimize_D = lasagne.updates.rmsprop(loss_D, params_netD, learning_rate=lrD)\n",
    "optimize_D = lasagne.updates.adam(loss_D, params_netD, learning_rate=lrD, beta1=0.0, beta2=0.9)\n",
    "train_G_fn =  theano.function([input_var_G], [loss_G], updates=optimize_G)\n",
    "\n",
    "train_D_fn = theano.function([input_var_D, input_var_G, ϵ], [loss_D, loss_D_real, loss_D_fake], \n",
    "                                         updates=optimize_D)\n",
    "generator_fn = theano.function([input_var_G], output_G)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fixed_noise = np.random.normal(size=(batchSize, nz)).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import time\n",
    "t0 = time.time()\n",
    "niter = 100\n",
    "gen_iterations = 0\n",
    "for epoch in range(niter):\n",
    "    i = 0\n",
    "    #  每個 epoch 洗牌一下\n",
    "    np.random.shuffle(train_X)\n",
    "    batches = train_X.shape[0]//batchSize\n",
    "    while i < batches:\n",
    "        if gen_iterations < 25 or gen_iterations %500 == 0:\n",
    "            _Diters = 100\n",
    "        else:\n",
    "            _Diters = Diters\n",
    "        j = 0\n",
    "        while j < _Diters and i < batches:\n",
    "            j+=1       \n",
    "            #clamp_D_fn()\n",
    "            real_data = train_X[i*batchSize:(i+1)*batchSize]\n",
    "            i+=1        \n",
    "            noise  = np.random.normal(size=(batchSize, nz)).astype('float32')\n",
    "            random_epsilon = real_data.std() * np.random.uniform(-0.5,0.5, size=real_data.shape) \n",
    "            random_epsilon *= np.random.uniform(size=(batchSize, 1,1,1))\n",
    "            random_epsilon = random_epsilon.astype('float32')\n",
    "            errD, errD_real, errD_fake = train_D_fn(real_data, noise, random_epsilon)\n",
    "        if gen_iterations%500 == 0:            \n",
    "            fake = generator_fn(fixed_noise)\n",
    "            showX(fake, 4)\n",
    "            \n",
    "        noise = np.random.normal(size=(batchSize, nz)).astype('float32')        \n",
    "        errG = train_G_fn(noise)[0]\n",
    "        if gen_iterations%500==0:\n",
    "            print('[%d/%d][%d/%d][%d] Loss_D: %f Loss_G: %f Loss_D_real: %f Loss_D_fake %f'\n",
    "            % (epoch, niter, i, batches, gen_iterations,\n",
    "            errD, errG, errD_real, errD_fake), time.time()-t0)\n",
    "\n",
    "        gen_iterations+=1\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}