u_net.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers import Input, concatenate, Conv2D, MaxPooling2D, BatchNormalization, Activation, UpSampling2D, Average, Add, SeparableConv2D, Conv2DTranspose\n",
    "from keras.optimizers import SGD, Adam\n",
    "from keras.losses import binary_crossentropy\n",
    "import keras.backend as K\n",
    "from keras import regularizers\n",
    "#from keras.layers import BatchNormalization, Convolution2D, Input, merge\n",
    "from keras.layers.core import Activation, Layer, SpatialDropout2D\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from keras.layers.advanced_activations import PReLU, ELU\n",
    "#from keras_contrib.layers import CRF\n",
    "#from keras.preprocessing.sequence import pad_sequences\n",
    "import numpy as np\n",
    "#np.random.seed(98643)\n",
    "import tensorflow as tf\n",
    "#tf.set_random_seed(683)\n",
    "#from crfrnn_layer import CrfRnnLayer\n",
    "from keras.layers.core import Dropout, Lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dice_coeff(y_true, y_pred):\n",
    "    smooth = 1.\n",
    "    y_true_f = K.flatten(y_true)\n",
    "    y_pred_f = K.flatten(y_pred)\n",
    "    intersection = K.sum(y_true_f * y_pred_f)\n",
    "    score = (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n",
    "    return score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dice_loss(y_true, y_pred):\n",
    "    loss = 1 - dice_coeff(y_true, y_pred)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bce_dice_loss(y_true, y_pred):\n",
    "    loss = binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)\n",
    "    return loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weighted_dice_coeff(y_true, y_pred, weight):\n",
    "    smooth = 1.\n",
    "    w, m1, m2 = weight * weight, y_true, y_pred\n",
    "    intersection = (m1 * m2)\n",
    "    score = (2. * K.sum(w * intersection) + smooth) / (K.sum(w * m1) + K.sum(w * m2) + smooth)\n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weighted_dice_loss(y_true, y_pred):\n",
    "    y_true = K.cast(y_true, 'float32')\n",
    "    y_pred = K.cast(y_pred, 'float32')\n",
    "    # if we want to get same size of output, kernel size must be odd number\n",
    "    if K.int_shape(y_pred)[1] == 128:\n",
    "        kernel_size = 11\n",
    "    elif K.int_shape(y_pred)[1] == 256:\n",
    "        kernel_size = 21\n",
    "    elif K.int_shape(y_pred)[1] == 512:\n",
    "        kernel_size = 21\n",
    "    elif K.int_shape(y_pred)[1] == 1024:\n",
    "        kernel_size = 41\n",
    "    else:\n",
    "        raise ValueError('Unexpected image size')\n",
    "    averaged_mask = K.pool2d(\n",
    "        y_true, pool_size=(kernel_size, kernel_size), strides=(1, 1), padding='same', pool_mode='avg')\n",
    "    border = K.cast(K.greater(averaged_mask, 0.005), 'float32') * K.cast(K.less(averaged_mask, 0.995), 'float32')\n",
    "    weight = K.ones_like(averaged_mask)\n",
    "    w0 = K.sum(weight)\n",
    "    weight += border * 2\n",
    "    w1 = K.sum(weight)\n",
    "    weight *= (w0 / w1)\n",
    "    loss = 1 - weighted_dice_coeff(y_true, y_pred, weight)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weighted_bce_loss(y_true, y_pred, weight):\n",
    "    # avoiding overflow\n",
    "    epsilon = 1e-7\n",
    "    y_pred = K.clip(y_pred, epsilon, 1. - epsilon)\n",
    "    logit_y_pred = K.log(y_pred / (1. - y_pred))\n",
    "\n",
    "    # https://www.tensorflow.org/api_docs/python/tf/nn/weighted_cross_entropy_with_logits\n",
    "    loss = (1. - y_true) * logit_y_pred + (1. + (weight - 1.) * y_true) * \\\n",
    "                                          (K.log(1. + K.exp(-K.abs(logit_y_pred))) + K.maximum(-logit_y_pred, 0.))\n",
    "    return K.sum(loss) / K.sum(weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weighted_bce_dice_loss(y_true, y_pred):\n",
    "    y_true = K.cast(y_true, 'float32')\n",
    "    y_pred = K.cast(y_pred, 'float32')\n",
    "    # if we want to get same size of output, kernel size must be odd number\n",
    "    if K.int_shape(y_pred)[1] == 128:\n",
    "        kernel_size = 11\n",
    "    elif K.int_shape(y_pred)[1] == 256:\n",
    "        kernel_size = 21\n",
    "    elif K.int_shape(y_pred)[1] == 512:\n",
    "        kernel_size = 21\n",
    "    elif K.int_shape(y_pred)[1] == 1024:\n",
    "        kernel_size = 41\n",
    "    else:\n",
    "        raise ValueError('Unexpected image size')\n",
    "    averaged_mask = K.pool2d(\n",
    "        y_true, pool_size=(kernel_size, kernel_size), strides=(1, 1), padding='same', pool_mode='avg')\n",
    "    border = K.cast(K.greater(averaged_mask, 0.005), 'float32') * K.cast(K.less(averaged_mask, 0.995), 'float32')\n",
    "    weight = K.ones_like(averaged_mask)\n",
    "    w0 = K.sum(weight)\n",
    "    weight += border * 2\n",
    "    w1 = K.sum(weight)\n",
    "    weight *= (w0 / w1)\n",
    "    loss = weighted_bce_loss(y_true, y_pred, weight) + (1 - weighted_dice_coeff(y_true, y_pred, weight))\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define IoU metric\n",
    "def mean_iou(y_true, y_pred):\n",
    "    prec = []\n",
    "    for t in np.arange(0.5, 1.0, 0.05):\n",
    "        y_pred_ = tf.to_int32(y_pred > t)\n",
    "        score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2)\n",
    "        K.get_session().run(tf.local_variables_initializer())\n",
    "        with tf.control_dependencies([up_opt]):\n",
    "            score = tf.identity(score)\n",
    "        prec.append(score)\n",
    "    return K.mean(K.stack(prec), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def inception_block(inputs, depth, splitted=False, activation='relu'):\n",
    "    assert depth % 16 == 0\n",
    "    actv = activation == 'relu' and (lambda: LeakyReLU(0.0)) or activation == 'elu' and (lambda: ELU(1.0)) or None\n",
    "\n",
    "    c1_1 = Conv2D(filters = int(depth / 4), kernel_size = (1, 1), init='he_normal', border_mode='same')(inputs)\n",
    "\n",
    "    c2_1 = Conv2D(filters = int(depth / 8 * 3), kernel_size = (1, 1), init='he_normal', border_mode='same')(inputs)\n",
    "    c2_1 = actv()(c2_1)\n",
    "    if splitted:\n",
    "        c2_2 = Conv2D(filters = int(depth / 2), kernel_size = (1, 3), init='he_normal', border_mode='same')(c2_1)\n",
    "        c2_2 = BatchNormalization()(c2_2)\n",
    "        c2_2 = actv()(c2_2)\n",
    "        c2_3 = Conv2D(filters = int(depth / 2), kernel_size = (3, 1), init='he_normal', border_mode='same')(c2_2)\n",
    "    else:\n",
    "        c2_3 = Conv2D(filters=int(depth / 2), kernel_size=(3, 3), init='he_normal', border_mode='same')(c2_1)\n",
    "\n",
    "    c3_1 = Conv2D(filters=int(depth / 16), kernel_size=(1, 1), init='he_normal', border_mode='same')(inputs)\n",
    "    # missed batch norm\n",
    "    c3_1 = actv()(c3_1)\n",
    "    if splitted:\n",
    "        c3_2 = Conv2D(filters=int(depth / 8), kernel_size=(1, 5), init='he_normal', border_mode='same')(c3_1)\n",
    "        c3_2 = BatchNormalization()(c3_2)\n",
    "        c3_2 = actv()(c3_2)\n",
    "        c3_3 = Conv2D(filters=int(depth / 8), kernel_size=(5, 1), init='he_normal', border_mode='same')(c3_2)\n",
    "    else:\n",
    "        c3_3 = Conv2D(filters=int(depth / 8), kernel_size=(5, 5), init='he_normal', border_mode='same')(c3_1)\n",
    "\n",
    "    p4_1 = MaxPooling2D(pool_size=(3, 3), strides=(1, 1), border_mode='same')(inputs)\n",
    "    c4_2 = Conv2D(filters=int(depth / 8), kernel_size=(1, 1), init='he_normal', border_mode='same')(p4_1)\n",
    "\n",
    "    res = concatenate([c1_1, c2_3, c3_3, c4_2], axis=3)\n",
    "    res = BatchNormalization()(res)\n",
    "    res = actv()(res)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def global_conv_block(inputs, k=31, channel=16):\n",
    "\n",
    "    feature_1 = Conv2D(channel, (k, 1), activation='relu', padding='same', dilation_rate=(1, 1))(inputs)\n",
    "    feature_1 = BatchNormalization()(feature_1)\n",
    "    feature_1 = Conv2D(channel, (1, k), activation='relu', padding='same', dilation_rate=(1, 1))(feature_1)\n",
    "    feature_1 = BatchNormalization()(feature_1)\n",
    "\n",
    "    feature_2 = Conv2D(channel, (1, k), activation='relu', padding='same', dilation_rate=(1, 1))(inputs)\n",
    "    feature_2 = BatchNormalization()(feature_2)\n",
    "    feature_2 = Conv2D(channel, (k, 1), activation='relu', padding='same', dilation_rate=(1, 1))(feature_2)\n",
    "    feature_2 = BatchNormalization()(feature_2)\n",
    "\n",
    "    feature = Add()([feature_1,feature_2])\n",
    "    return feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def boundary_refine(inputs, channel=16):\n",
    "    br = Conv2D(channel, (3, 3), padding='same', use_bias=True)(inputs)\n",
    "    br = BatchNormalization()(br)\n",
    "    br = Activation('relu')(br)\n",
    "    br = Conv2D(channel, (3, 3), padding='same', use_bias=True)(br)\n",
    "    return br\n",
    "\n",
    "########################################################################################\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_unet_128(input_shape=(128, 128, 3),\n",
    "                 num_classes=1):\n",
    "    inputs = Input(shape=input_shape)\n",
    "    # 128\n",
    "\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(inputs)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)\n",
    "    # 64\n",
    "\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down1_pool)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)\n",
    "    # 32\n",
    "\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down2_pool)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)\n",
    "    # 16\n",
    "\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down3_pool)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)\n",
    "    # 8\n",
    "\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(down4_pool)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    # center\n",
    "\n",
    "    up4 = UpSampling2D((2, 2))(center)\n",
    "    up4 = concatenate([down4, up4], axis=3)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    # 16\n",
    "\n",
    "    up3 = UpSampling2D((2, 2))(up4)\n",
    "    up3 = concatenate([down3, up3], axis=3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    # 32\n",
    "\n",
    "    up2 = UpSampling2D((2, 2))(up3)\n",
    "    up2 = concatenate([down2, up2], axis=3)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    # 64\n",
    "\n",
    "    up1 = UpSampling2D((2, 2))(up2)\n",
    "    up1 = concatenate([down1, up1], axis=3)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    # 128\n",
    "\n",
    "    classify = Conv2D(num_classes, (1, 1), activation='sigmoid')(up1)\n",
    "\n",
    "    model = Model(inputs=inputs, outputs=classify)\n",
    "    model.compile(optimizer=Adam(lr=0.01), loss='binary_crossentropy', metrics=[dice_loss])\n",
    "    #model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss='binary_crossentropy', metrics=[dice_loss])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_unet_256(input_shape=(256, 256, 3),\n",
    "                 num_classes=1):\n",
    "    inputs = Input(shape=input_shape)\n",
    "    # 256\n",
    "\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(inputs)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)\n",
    "    # 128\n",
    "\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down0_pool)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)\n",
    "    # 64\n",
    "\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down1_pool)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)\n",
    "    # 32\n",
    "\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down2_pool)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)\n",
    "    # 16\n",
    "\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down3_pool)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)\n",
    "    # 8\n",
    "\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(down4_pool)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    # center\n",
    "\n",
    "    up4 = UpSampling2D((2, 2))(center)\n",
    "    up4 = concatenate([down4, up4], axis=3)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    # 16\n",
    "\n",
    "    up3 = UpSampling2D((2, 2))(up4)\n",
    "    up3 = concatenate([down3, up3], axis=3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    # 32\n",
    "\n",
    "    up2 = UpSampling2D((2, 2))(up3)\n",
    "    up2 = concatenate([down2, up2], axis=3)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    # 64\n",
    "\n",
    "    up1 = UpSampling2D((2, 2))(up2)\n",
    "    up1 = concatenate([down1, up1], axis=3)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    # 128\n",
    "\n",
    "    up0 = UpSampling2D((2, 2))(up1)\n",
    "    up0 = concatenate([down0, up0], axis=3)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    # 256\n",
    "\n",
    "    classify = Conv2D(num_classes, (1, 1), activation='sigmoid')(up0)\n",
    "\n",
    "    model = Model(inputs=inputs, outputs=classify)\n",
    "\n",
    "    model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss='binary_crossentropy', metrics=[dice_loss])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_unet_1024(input_shape=(1024, 1024, 3),\n",
    "                  num_classes=1):\n",
    "    inputs = Input(shape=input_shape)\n",
    "    # 1024\n",
    "\n",
    "    down0b = Conv2D(8, (3, 3), padding='same')(inputs)\n",
    "    down0b = BatchNormalization()(down0b)\n",
    "    down0b = Activation('relu')(down0b)\n",
    "    down0b = Conv2D(8, (3, 3), padding='same')(down0b)\n",
    "    down0b = BatchNormalization()(down0b)\n",
    "    down0b = Activation('relu')(down0b)\n",
    "    down0b_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0b)\n",
    "    # 512\n",
    "\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(down0b_pool)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(down0a)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    down0a_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0a)\n",
    "    # 256\n",
    "\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0a_pool)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)\n",
    "    # 128\n",
    "\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down0_pool)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)\n",
    "    # 64\n",
    "\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down1_pool)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)\n",
    "    # 32\n",
    "\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down2_pool)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)\n",
    "    # 16\n",
    "\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down3_pool)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)\n",
    "    # 8\n",
    "\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(down4_pool)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    # center\n",
    "\n",
    "    up4 = UpSampling2D((2, 2))(center)\n",
    "    up4 = concatenate([down4, up4], axis=3)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    # 16\n",
    "\n",
    "    up3 = UpSampling2D((2, 2))(up4)\n",
    "    up3 = concatenate([down3, up3], axis=3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    # 32\n",
    "\n",
    "    up2 = UpSampling2D((2, 2))(up3)\n",
    "    up2 = concatenate([down2, up2], axis=3)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    # 64\n",
    "\n",
    "    up1 = UpSampling2D((2, 2))(up2)\n",
    "    up1 = concatenate([down1, up1], axis=3)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    # 128\n",
    "\n",
    "    up0 = UpSampling2D((2, 2))(up1)\n",
    "    up0 = concatenate([down0, up0], axis=3)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    # 256\n",
    "\n",
    "    up0a = UpSampling2D((2, 2))(up0)\n",
    "    up0a = concatenate([down0a, up0a], axis=3)\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    # 512\n",
    "\n",
    "    up0b = UpSampling2D((2, 2))(up0a)\n",
    "    up0b = concatenate([down0b, up0b], axis=3)\n",
    "    up0b = Conv2D(8, (3, 3), padding='same')(up0b)\n",
    "    up0b = BatchNormalization()(up0b)\n",
    "    up0b = Activation('relu')(up0b)\n",
    "    up0b = Conv2D(8, (3, 3), padding='same')(up0b)\n",
    "    up0b = BatchNormalization()(up0b)\n",
    "    up0b = Activation('relu')(up0b)\n",
    "    up0b = Conv2D(8, (3, 3), padding='same')(up0b)\n",
    "    up0b = BatchNormalization()(up0b)\n",
    "    up0b = Activation('relu')(up0b)\n",
    "    # 1024\n",
    "\n",
    "    classify = Conv2D(num_classes, (1, 1), activation='sigmoid')(up0b)\n",
    "\n",
    "    model = Model(inputs=inputs, outputs=classify)\n",
    "\n",
    "    model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss=bce_dice_loss, metrics=[dice_loss])\n",
    "\n",
    "    return model\n",
    "\n",
    "########################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_funet_640_960(input_shape=(640, 960, 3),\n",
    "                 num_classes=1):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "\n",
    "    # ===== Fractal block ==========\n",
    "\n",
    "    # blk 1 640\n",
    "    x11 = Conv2D(16, (3, 3), padding='same', kernel_initializer='he_normal')(inputs)\n",
    "    x11 = BatchNormalization()(x11)\n",
    "    x21 = Conv2D(16, (3, 3), padding='same', kernel_initializer='he_normal')(x11)\n",
    "    x21 = BatchNormalization()(x21)\n",
    "    avg1 = Average()([x11, x21])\n",
    "    x31 = Conv2D(16, (3, 3), padding='same', kernel_initializer='he_normal')(avg1)\n",
    "    x31 = BatchNormalization()(x31)\n",
    "    x41 = Conv2D(16, (3, 3), padding='same', kernel_initializer='he_normal')(x31)\n",
    "    x41 = BatchNormalization()(x41)\n",
    "    blk1 = Average()([x11, x31, x41])\n",
    "\n",
    "    blk1 = Activation('relu')(blk1)\n",
    "    blk1_pool = MaxPooling2D((2, 2), strides=(2, 2))(blk1)\n",
    "\n",
    "    # blk 2 320\n",
    "    x12 = Conv2D(32, (3, 3), padding='same', kernel_initializer='he_normal')(blk1_pool)\n",
    "    x12 = BatchNormalization()(x12)\n",
    "    x22 = Conv2D(32, (3, 3), padding='same', kernel_initializer='he_normal')(x12)\n",
    "    x22 = BatchNormalization()(x22)\n",
    "    avg2 = Average()([x12, x22])\n",
    "    x32 = Conv2D(32, (3, 3), padding='same', kernel_initializer='he_normal')(avg2)\n",
    "    x32 = BatchNormalization()(x32)\n",
    "    x42 = Conv2D(32, (3, 3), padding='same', kernel_initializer='he_normal')(x32)\n",
    "    x42 = BatchNormalization()(x42)\n",
    "    blk2 = Average()([x12, x32, x42])\n",
    "\n",
    "    blk2 = Activation('relu')(blk2)\n",
    "    blk2_pool = MaxPooling2D((2, 2), strides=(2, 2))(blk2)\n",
    "\n",
    "    # blk 3 160\n",
    "    x13 = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal')(blk2_pool)\n",
    "    x13 = BatchNormalization()(x13)\n",
    "    x23 = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal')(x13)\n",
    "    x23 = BatchNormalization()(x23)\n",
    "    avg3 = Average()([x13, x23])\n",
    "    x33 = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal')(avg3)\n",
    "    x33 = BatchNormalization()(x33)\n",
    "    x43 = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal')(x33)\n",
    "    x43 = BatchNormalization()(x43)\n",
    "    blk3 = Average()([x13, x33, x43])\n",
    "\n",
    "    blk3 = Activation('relu')(blk3)\n",
    "    blk3_pool = MaxPooling2D((2, 2), strides=(2, 2))(blk3)\n",
    "\n",
    "    # Center 80\n",
    "    center = Conv2D(128, (3, 3), padding='same')(blk3_pool)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(128, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "\n",
    "    # blk up 3 160\n",
    "    blk3u = UpSampling2D((2, 2))(center)\n",
    "    blk3u = concatenate([blk3, blk3u], axis=3)\n",
    "\n",
    "    y13 = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal')(blk3u)\n",
    "    y13 = BatchNormalization()(y13)\n",
    "    y23 = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal')(y13)\n",
    "    y23 = BatchNormalization()(y23)\n",
    "    avg3u = Average()([y13, y23])\n",
    "    y33 = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal')(avg3u)\n",
    "    y33 = BatchNormalization()(y33)\n",
    "    y43 = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal')(y33)\n",
    "    y43 = BatchNormalization()(y43)\n",
    "    blk3u = Average()([y13, y33, y43])\n",
    "\n",
    "    blk3u = Activation('relu')(blk3u)\n",
    "\n",
    "    # blk up 2 320\n",
    "    blk2u = UpSampling2D((2, 2))(blk3u)\n",
    "    blk2u = concatenate([blk2, blk2u], axis=3)\n",
    "\n",
    "    y12 = Conv2D(32, (3, 3), padding='same', kernel_initializer='he_normal')(blk2u)\n",
    "    y12 = BatchNormalization()(y12)\n",
    "    y22 = Conv2D(32, (3, 3), padding='same', kernel_initializer='he_normal')(y12)\n",
    "    y22 = BatchNormalization()(y22)\n",
    "    avg2u = Average()([y12, y22])\n",
    "    y32 = Conv2D(32, (3, 3), padding='same', kernel_initializer='he_normal')(avg2u)\n",
    "    y32 = BatchNormalization()(y32)\n",
    "    y42 = Conv2D(32, (3, 3), padding='same', kernel_initializer='he_normal')(y32)\n",
    "    y42 = BatchNormalization()(y42)\n",
    "    blk2u = Average()([y12, y32, y42])\n",
    "\n",
    "    blk2u = Activation('relu')(blk2u)\n",
    "\n",
    "    # blk up 1 640\n",
    "    blk1u = UpSampling2D((2, 2))(blk2u)\n",
    "    blk1u = concatenate([blk1, blk1u], axis=3)\n",
    "\n",
    "    y11 = Conv2D(16, (3, 3), padding='same', kernel_initializer='he_normal')(blk1u)\n",
    "    y11 = BatchNormalization()(y11)\n",
    "    y21 = Conv2D(16, (3, 3), padding='same', kernel_initializer='he_normal')(y11)\n",
    "    y21 = BatchNormalization()(y21)\n",
    "    avg1u = Average()([y11, y21])\n",
    "    y31 = Conv2D(16, (3, 3), padding='same', kernel_initializer='he_normal')(avg1u)\n",
    "    y31 = BatchNormalization()(y31)\n",
    "    y41 = Conv2D(16, (3, 3), padding='same', kernel_initializer='he_normal')(y31)\n",
    "    y41 = BatchNormalization()(y41)\n",
    "    blk1u = Average()([y11, y31, y41])\n",
    "\n",
    "    blk1u = Activation('relu')(blk1u)\n",
    "\n",
    "    classify = Conv2D(num_classes, (1, 1), activation='sigmoid')(blk1u)\n",
    "\n",
    "    model = Model(inputs=inputs, outputs=classify)\n",
    "\n",
    "    model.compile(optimizer=Adam(lr=0.01), loss='binary_crossentropy', metrics=[dice_loss])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_dialatedNet_640_960(input_shape=(640, 960, 3),\n",
    "                       num_classes=1):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Block 1\n",
    "    x = Conv2D(24, (3, 3), activation='relu', padding='same', dilation_rate=(1, 1), name='block1_conv1')(inputs)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Conv2D(24, (3, 3), activation='relu', padding='same', dilation_rate=(1, 1),name='block1_conv2')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    blk1 = x\n",
    "\n",
    "\n",
    "\n",
    "    # Block 2\n",
    "    x = Conv2D(32, (3, 3), activation='relu', padding='same', dilation_rate=(2, 2), name='block2_conv1')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Conv2D(32, (3, 3), activation='relu', padding='same', dilation_rate=(2, 2),name='block2_conv2')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    blk2 = x\n",
    "\n",
    "    # Block 3\n",
    "    x = Conv2D(64, (3, 3), activation='relu', padding='same', dilation_rate=(4, 4),name='block3_conv1')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Conv2D(64, (3, 3), activation='relu', padding='same', dilation_rate=(4, 4),name='block3_conv2')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    #x = Conv2D(64, (3, 3), activation='relu', padding='same', dilation_rate=(4, 4),name='block3_conv3')(x)\n",
    "    #x = BatchNormalization()(x)\n",
    "    blk3 = x\n",
    "\n",
    "\n",
    "    # Block 4\n",
    "    x = Conv2D(64, (3, 3), activation='relu', padding='same', dilation_rate=(8, 8),name='block4_conv1')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Conv2D(64, (3, 3), activation='relu', padding='same', dilation_rate=(8, 8),name='block4_conv2')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    #x = Conv2D(128, (3, 3), activation='relu', padding='same', dilation_rate=(8, 8),name='block4_conv3')(x)\n",
    "    #x = BatchNormalization()(x)\n",
    "    blk4 = x\n",
    "\n",
    "    # Block 5\n",
    "    x = Conv2D(8, (3, 3), activation='relu', padding='same', dilation_rate=(16, 16),name='block5_conv1')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Conv2D(8, (3, 3), activation='relu', padding='same', dilation_rate=(16, 16),name='block5_conv2')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Conv2D(8, (3, 3), activation='relu', padding='same', dilation_rate=(16, 16),name='block5_conv3')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    blk5 = x\n",
    "\n",
    "    x = concatenate([blk1, blk2, blk3, blk4, blk5])\n",
    "\n",
    "    classify = Conv2D(num_classes, (1, 1), activation='sigmoid', dilation_rate=(1, 1), name='classification')(x)\n",
    "\n",
    "    model = Model(inputs=inputs, outputs=classify)\n",
    "\n",
    "    for i, layer in enumerate(model.layers):\n",
    "        print(i, layer.name)\n",
    "\n",
    "    model.summary()\n",
    "\n",
    "    # model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss=bce_dice_loss, metrics=[dice_loss])\n",
    "    model.compile(optimizer=Adam(lr=0.01), loss=bce_dice_loss, metrics=[dice_coeff])\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "#########################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_unet_640_960_8(input_shape=(640, 960, 3),\n",
    "                       num_classes=1):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Res block\n",
    "\n",
    "    input_conv = Conv2D(24, (3, 3), padding='same')(inputs)\n",
    "    input_conv = BatchNormalization()(input_conv)\n",
    "    input_conv= Activation('relu')(input_conv)\n",
    "\n",
    "    # 640x960 -\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(inputs)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower1 = down0a\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(down0a)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower2 = down0a\n",
    "    down0a = concatenate([tower1, tower2], axis=3)\n",
    "    down0a = Conv2D(24, (1, 1), padding='same')(down0a)\n",
    "    down0a_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0a)\n",
    "\n",
    "    res_stream = Add()([down0a, input_conv]) #640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0a_pool)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower1 = down0\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower2 = down0\n",
    "    down0 = concatenate([tower1, tower2], axis=3)\n",
    "    down0 = Conv2D(24, (1, 1), padding='same')(down0)\n",
    "    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)\n",
    "\n",
    "    res0 = UpSampling2D((2,2))(down0)\n",
    "    res_stream = Add()([res0, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 160x240\n",
    "\n",
    "    res_stream_pool =  MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    down1 = concatenate([down0_pool, res_stream_pool], axis=3)\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower1 = down1\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower2 = down1\n",
    "    down1 = concatenate([tower1, tower2], axis=3)\n",
    "    down1 = Conv2D(24, (1, 1), padding='same')(down1)\n",
    "    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)\n",
    "\n",
    "    res1 = UpSampling2D((4,4))(down1) #640\n",
    "    res_stream = Add()([res1, res_stream])\n",
    "\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool =  MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    down2 = concatenate([down1_pool, res_stream_pool], axis=3)\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower1 = down2\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower2 = down2\n",
    "    down2 = concatenate([tower1, tower2], axis=3)\n",
    "    down2 = Conv2D(24, (1, 1), padding='same')(down2)\n",
    "    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)\n",
    "\n",
    "    res2 = UpSampling2D((8,8))(down2)\n",
    "    res_stream = Add()([res2, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "    down3 = concatenate([down2_pool, res_stream_pool], axis=3)\n",
    "\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower1 = down3\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower2 = down3\n",
    "    down3 = concatenate([tower1, tower2], axis=3)\n",
    "    down3 = Conv2D(24, (1, 1), padding='same')(down3)\n",
    "    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)\n",
    "\n",
    "    res3 = UpSampling2D((16,16))(down3)\n",
    "    res_stream = Add()([res3, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    down4 = concatenate([down3_pool, res_stream_pool], axis=3)\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower1 = down4\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower2 = down4\n",
    "    down4 = concatenate([tower1, tower2], axis=3)\n",
    "    down4 = Conv2D(24, (1, 1), padding='same')(down4)\n",
    "    down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)\n",
    "\n",
    "    res4 = UpSampling2D((32,32))(down4)\n",
    "    res_stream = Add()([res4, res_stream])  #640\n",
    "\n",
    "\n",
    "    # 10x15\n",
    "    res_stream_pool = MaxPooling2D((64, 64), strides=(64, 64))(res_stream)\n",
    "\n",
    "    center = concatenate([down4_pool, res_stream_pool], axis=3)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(24, (1, 1), padding='same')(center)\n",
    "\n",
    "    resCen = UpSampling2D((64,64))(center)\n",
    "    res_stream = Add()([resCen, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    up4 = UpSampling2D((2, 2))(center)\n",
    "    up4 = concatenate([res_stream_pool, up4], axis=3)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower1 = up4\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower2 = up4\n",
    "    up4 = concatenate([tower1,tower2], axis=3)\n",
    "    up4 = Conv2D(24, (1, 1), padding='same')(up4)\n",
    "\n",
    "    res_up4 = UpSampling2D((32, 32))(up4)\n",
    "    res_stream = Add()([res_up4, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "\n",
    "    up3 = UpSampling2D((2, 2))(up4)\n",
    "    up3 = concatenate([res_stream_pool, up3], axis=3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower1 = up3\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower2 = up3\n",
    "    up3 = concatenate([tower1, tower2], axis=3)\n",
    "    up3 = Conv2D(24, (1, 1), padding='same')(up3)\n",
    "\n",
    "    res_up3 = UpSampling2D((16, 16))(up3)\n",
    "    res_stream = Add()([res_up3, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool = MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    up2 = UpSampling2D((2, 2))(up3)\n",
    "    up2 = concatenate([res_stream_pool, up2], axis=3)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower1 = up2\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower2 = up2\n",
    "    up2 = concatenate([tower1,tower2], axis=3)\n",
    "    up2 = Conv2D(24, (1, 1), padding='same')(up2)\n",
    "\n",
    "    res_up2 = UpSampling2D((8, 8))(up2)\n",
    "    res_stream = Add()([res_up2, res_stream])  # 640\n",
    "\n",
    "    # 160x240\n",
    "    res_stream_pool = MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    up1 = UpSampling2D((2, 2))(up2)\n",
    "    up1 = concatenate([res_stream_pool, up1], axis=3)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower1 = up1\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower2 = up1\n",
    "    up1 = concatenate([tower1,tower2],axis=3)\n",
    "    up1 = Conv2D(24, (1, 1), padding='same')(up1)\n",
    "\n",
    "    res_up1 = UpSampling2D((4, 4))(up1)\n",
    "    res_stream = Add()([res_up1, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    res_stream_pool = MaxPooling2D((2, 2), strides=(2, 2))(res_stream)\n",
    "\n",
    "    up0 = UpSampling2D((2, 2))(up1)\n",
    "    up0 = concatenate([res_stream_pool, up0, down0], axis=3)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower1 = up0\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower2 = up0\n",
    "    up0 = concatenate([tower1,tower2],axis=3)\n",
    "    up0 = Conv2D(24, (1, 1), padding='same')(up0)\n",
    "\n",
    "    res_up0 = UpSampling2D((2, 2))(up0)\n",
    "    res_stream = Add()([res_up0, res_stream])  # 640\n",
    "\n",
    "    # 640x960\n",
    "\n",
    "    up0a = UpSampling2D((2, 2))(up0)\n",
    "    up0a = concatenate([res_stream, down0a, up0a], axis=3)\n",
    "\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower1 = up0a\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower2 = up0a\n",
    "    up0a = concatenate([tower1,tower2],axis=3)\n",
    "    up0a = Conv2D(24, (1, 1), padding='same')(up0a)\n",
    "    #up0a = SpatialDropout2D(0.5)(up0a)\n",
    "\n",
    "    res_stream = Add()([up0a, res_stream])  # 640\n",
    "    #keras.layers.convolutional.Conv2D(filters, kernel_size, strides=(1, 1), padding='valid', data_format=None,\n",
    "    #                                  dilation_rate=(1, 1), activation=None, use_bias=True,\n",
    "    #                                  kernel_initializer='glorot_uniform', bias_initializer='zeros',\n",
    "    #                                  kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None,\n",
    "    #                                  kernel_constraint=None, bias_constraint=None)\n",
    "\n",
    "    features = Conv2D(24, (3, 3), padding='same', use_bias=False)(res_stream)\n",
    "    classify = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True)(features)\n",
    "\n",
    "    #8 net experiment\n",
    "    #features2 = Conv2D(8, (5, 5), padding='same', use_bias=True, name = 'featurs2')(res_stream)\n",
    "    #classify2 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify2')(features2)\n",
    "    #features3 = Conv2D(8, (7, 7), padding='same', use_bias=True, name = 'featurs3')(res_stream)\n",
    "    #classify3 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify3')(features3)\n",
    "\n",
    "    #main_feature = Average()([classify1,classify2,classify3])\n",
    "    #main_output = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='main_output')(main_feature)\n",
    "\n",
    "    ##crf = CRF( units=128,sparse_target=True, activation='sigmoid',name = 'crf')(classify) # for word embedding. one dimentional\n",
    "\n",
    "    #output = CrfRnnLayer(image_dims=(640, 960),\n",
    "    #                     num_classes=1,\n",
    "    #                     theta_alpha=160.,\n",
    "    #                     theta_beta=3.,\n",
    "    #                     theta_gamma=3.,\n",
    "    #                     num_iterations=5,\n",
    "    #                     name='crfrnn')([classify, inputs])\n",
    "\n",
    "    #model = Model(inputs=inputs, outputs=[output,classify])\n",
    "    model = Model(inputs=inputs, outputs=classify)\n",
    "    #model = Model(inputs=inputs, outputs=[classify,tower1,tower2,tower3])\n",
    "\n",
    "    for i, layer in enumerate(model.layers):\n",
    "        print(i, layer.name)\n",
    "\n",
    "    model.summary()\n",
    "\n",
    "    # model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss=bce_dice_loss, metrics=[dice_loss])\n",
    "    #model.compile(optimizer=Adam(lr=0.0001), loss=bce_dice_loss, metrics=[dice_coeff])\n",
    "\n",
    "    model.compile(optimizer=Adam(lr=0.0005),\n",
    "                  loss=bce_dice_loss,\n",
    "                  #loss={'main_output': bce_dice_loss, 'aux_output': 'binary_crossentropy'},\n",
    "                  metrics=[dice_coeff])\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5},\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5})\n",
    "\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_unet_1024_8(input_shape=(1024, 1024, 3),\n",
    "                       num_classes=1):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Res block\n",
    "\n",
    "    input_conv = Conv2D(24, (3, 3), padding='same')(inputs)\n",
    "    input_conv = BatchNormalization()(input_conv)\n",
    "    input_conv= Activation('relu')(input_conv)\n",
    "\n",
    "    # 640x960 -\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(inputs)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower1 = down0a\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(down0a)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower2 = down0a\n",
    "    down0a = concatenate([tower1, tower2], axis=3)\n",
    "    down0a = Conv2D(24, (1, 1), padding='same')(down0a)\n",
    "    down0a_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0a)\n",
    "\n",
    "    res_stream = Add()([down0a, input_conv]) #640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0a_pool)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower1 = down0\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower2 = down0\n",
    "    down0 = concatenate([tower1, tower2], axis=3)\n",
    "    down0 = Conv2D(24, (1, 1), padding='same')(down0)\n",
    "    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)\n",
    "\n",
    "    res0 = UpSampling2D((2,2))(down0)\n",
    "    res_stream = Add()([res0, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 160x240\n",
    "\n",
    "    res_stream_pool =  MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    down1 = concatenate([down0_pool, res_stream_pool], axis=3)\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower1 = down1\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower2 = down1\n",
    "    down1 = concatenate([tower1, tower2], axis=3)\n",
    "    down1 = Conv2D(24, (1, 1), padding='same')(down1)\n",
    "    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)\n",
    "\n",
    "    res1 = UpSampling2D((4,4))(down1) #640\n",
    "    res_stream = Add()([res1, res_stream])\n",
    "\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool =  MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    down2 = concatenate([down1_pool, res_stream_pool], axis=3)\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower1 = down2\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower2 = down2\n",
    "    down2 = concatenate([tower1, tower2], axis=3)\n",
    "    down2 = Conv2D(24, (1, 1), padding='same')(down2)\n",
    "    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)\n",
    "\n",
    "    res2 = UpSampling2D((8,8))(down2)\n",
    "    res_stream = Add()([res2, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "    down3 = concatenate([down2_pool, res_stream_pool], axis=3)\n",
    "\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower1 = down3\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower2 = down3\n",
    "    down3 = concatenate([tower1, tower2], axis=3)\n",
    "    down3 = Conv2D(24, (1, 1), padding='same')(down3)\n",
    "    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)\n",
    "\n",
    "    res3 = UpSampling2D((16,16))(down3)\n",
    "    res_stream = Add()([res3, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    down4 = concatenate([down3_pool, res_stream_pool], axis=3)\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower1 = down4\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower2 = down4\n",
    "    down4 = concatenate([tower1, tower2], axis=3)\n",
    "    down4 = Conv2D(24, (1, 1), padding='same')(down4)\n",
    "    down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)\n",
    "\n",
    "    res4 = UpSampling2D((32,32))(down4)\n",
    "    res_stream = Add()([res4, res_stream])  #640\n",
    "\n",
    "\n",
    "    # 10x15\n",
    "    res_stream_pool = MaxPooling2D((64, 64), strides=(64, 64))(res_stream)\n",
    "\n",
    "    center = concatenate([down4_pool, res_stream_pool], axis=3)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(24, (1, 1), padding='same')(center)\n",
    "\n",
    "    resCen = UpSampling2D((64,64))(center)\n",
    "    res_stream = Add()([resCen, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    up4 = UpSampling2D((2, 2))(center)\n",
    "    up4 = concatenate([res_stream_pool, up4], axis=3)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower1 = up4\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower2 = up4\n",
    "    up4 = concatenate([tower1,tower2], axis=3)\n",
    "    up4 = Conv2D(24, (1, 1), padding='same')(up4)\n",
    "\n",
    "    res_up4 = UpSampling2D((32, 32))(up4)\n",
    "    res_stream = Add()([res_up4, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "\n",
    "    up3 = UpSampling2D((2, 2))(up4)\n",
    "    up3 = concatenate([res_stream_pool, up3], axis=3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower1 = up3\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower2 = up3\n",
    "    up3 = concatenate([tower1, tower2], axis=3)\n",
    "    up3 = Conv2D(24, (1, 1), padding='same')(up3)\n",
    "\n",
    "    res_up3 = UpSampling2D((16, 16))(up3)\n",
    "    res_stream = Add()([res_up3, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool = MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    up2 = UpSampling2D((2, 2))(up3)\n",
    "    up2 = concatenate([res_stream_pool, up2], axis=3)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower1 = up2\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower2 = up2\n",
    "    up2 = concatenate([tower1,tower2], axis=3)\n",
    "    up2 = Conv2D(24, (1, 1), padding='same')(up2)\n",
    "\n",
    "    res_up2 = UpSampling2D((8, 8))(up2)\n",
    "    res_stream = Add()([res_up2, res_stream])  # 640\n",
    "\n",
    "    # 160x240\n",
    "    res_stream_pool = MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    up1 = UpSampling2D((2, 2))(up2)\n",
    "    up1 = concatenate([res_stream_pool, up1], axis=3)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower1 = up1\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower2 = up1\n",
    "    up1 = concatenate([tower1,tower2],axis=3)\n",
    "    up1 = Conv2D(24, (1, 1), padding='same')(up1)\n",
    "\n",
    "    res_up1 = UpSampling2D((4, 4))(up1)\n",
    "    res_stream = Add()([res_up1, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    res_stream_pool = MaxPooling2D((2, 2), strides=(2, 2))(res_stream)\n",
    "\n",
    "    up0 = UpSampling2D((2, 2))(up1)\n",
    "    up0 = concatenate([res_stream_pool, up0, down0], axis=3)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower1 = up0\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower2 = up0\n",
    "    up0 = concatenate([tower1,tower2],axis=3)\n",
    "    up0 = Conv2D(24, (1, 1), padding='same')(up0)\n",
    "\n",
    "    res_up0 = UpSampling2D((2, 2))(up0)\n",
    "    res_stream = Add()([res_up0, res_stream])  # 640\n",
    "\n",
    "    # 640x960\n",
    "\n",
    "    up0a = UpSampling2D((2, 2))(up0)\n",
    "    up0a = concatenate([res_stream, down0a, up0a], axis=3)\n",
    "\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower1 = up0a\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower2 = up0a\n",
    "    up0a = concatenate([tower1,tower2],axis=3)\n",
    "    up0a = Conv2D(24, (1, 1), padding='same')(up0a)\n",
    "    #up0a = SpatialDropout2D(0.5)(up0a)\n",
    "\n",
    "    res_stream = Add()([up0a, res_stream])  # 640\n",
    "    #keras.layers.convolutional.Conv2D(filters, kernel_size, strides=(1, 1), padding='valid', data_format=None,\n",
    "    #                                  dilation_rate=(1, 1), activation=None, use_bias=True,\n",
    "    #                                  kernel_initializer='glorot_uniform', bias_initializer='zeros',\n",
    "    #                                  kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None,\n",
    "    #                                  kernel_constraint=None, bias_constraint=None)\n",
    "\n",
    "    features = Conv2D(24, (3, 3), padding='same', use_bias=False)(res_stream)\n",
    "    classify = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True)(features)\n",
    "\n",
    "    #8 net experiment\n",
    "    #features2 = Conv2D(8, (5, 5), padding='same', use_bias=True, name = 'featurs2')(res_stream)\n",
    "    #classify2 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify2')(features2)\n",
    "    #features3 = Conv2D(8, (7, 7), padding='same', use_bias=True, name = 'featurs3')(res_stream)\n",
    "    #classify3 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify3')(features3)\n",
    "\n",
    "    #main_feature = Average()([classify1,classify2,classify3])\n",
    "    #main_output = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='main_output')(main_feature)\n",
    "\n",
    "    ##crf = CRF( units=128,sparse_target=True, activation='sigmoid',name = 'crf')(classify) # for word embedding. one dimentional\n",
    "\n",
    "    #output = CrfRnnLayer(image_dims=(640, 960),\n",
    "    #                     num_classes=1,\n",
    "    #                     theta_alpha=160.,\n",
    "    #                     theta_beta=3.,\n",
    "    #                     theta_gamma=3.,\n",
    "    #                     num_iterations=5,\n",
    "    #                     name='crfrnn')([classify, inputs])\n",
    "\n",
    "    #model = Model(inputs=inputs, outputs=[output,classify])\n",
    "    model = Model(inputs=inputs, outputs=classify)\n",
    "    #model = Model(inputs=inputs, outputs=[classify,tower1,tower2,tower3])\n",
    "\n",
    "    for i, layer in enumerate(model.layers):\n",
    "        print(i, layer.name)\n",
    "\n",
    "    model.summary()\n",
    "\n",
    "    # model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss=bce_dice_loss, metrics=[dice_loss])\n",
    "    #model.compile(optimizer=Adam(lr=0.0001), loss=bce_dice_loss, metrics=[dice_coeff])\n",
    "\n",
    "    model.compile(optimizer=Adam(lr=0.00015),\n",
    "                  loss=bce_dice_loss,\n",
    "                  #loss={'main_output': bce_dice_loss, 'aux_output': 'binary_crossentropy'},\n",
    "                  metrics=[dice_coeff])\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5},\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5})\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_unet_832x1216_8(input_shape=(832, 1216, 3),\n",
    "                       num_classes=1):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Res block\n",
    "\n",
    "    input_conv = Conv2D(24, (3, 3), padding='same')(inputs)\n",
    "    input_conv = BatchNormalization()(input_conv)\n",
    "    input_conv= Activation('relu')(input_conv)\n",
    "\n",
    "    # 640x960 -\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(inputs)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower1 = down0a\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(down0a)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower2 = down0a\n",
    "    down0a = concatenate([tower1, tower2], axis=3)\n",
    "    down0a = Conv2D(24, (1, 1), padding='same')(down0a)\n",
    "    down0a_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0a)\n",
    "\n",
    "    res_stream = Add()([down0a, input_conv]) #640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0a_pool)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower1 = down0\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower2 = down0\n",
    "    down0 = concatenate([tower1, tower2], axis=3)\n",
    "    down0 = Conv2D(24, (1, 1), padding='same')(down0)\n",
    "    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)\n",
    "\n",
    "    res0 = UpSampling2D((2,2))(down0)\n",
    "    res_stream = Add()([res0, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 160x240\n",
    "\n",
    "    res_stream_pool =  MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    down1 = concatenate([down0_pool, res_stream_pool], axis=3)\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower1 = down1\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower2 = down1\n",
    "    down1 = concatenate([tower1, tower2], axis=3)\n",
    "    down1 = Conv2D(24, (1, 1), padding='same')(down1)\n",
    "    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)\n",
    "\n",
    "    res1 = UpSampling2D((4,4))(down1) #640\n",
    "    res_stream = Add()([res1, res_stream])\n",
    "\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool =  MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    down2 = concatenate([down1_pool, res_stream_pool], axis=3)\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower1 = down2\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower2 = down2\n",
    "    down2 = concatenate([tower1, tower2], axis=3)\n",
    "    down2 = Conv2D(24, (1, 1), padding='same')(down2)\n",
    "    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)\n",
    "\n",
    "    res2 = UpSampling2D((8,8))(down2)\n",
    "    res_stream = Add()([res2, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "    down3 = concatenate([down2_pool, res_stream_pool], axis=3)\n",
    "\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower1 = down3\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower2 = down3\n",
    "    down3 = concatenate([tower1, tower2], axis=3)\n",
    "    down3 = Conv2D(24, (1, 1), padding='same')(down3)\n",
    "    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)\n",
    "\n",
    "    res3 = UpSampling2D((16,16))(down3)\n",
    "    res_stream = Add()([res3, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    down4 = concatenate([down3_pool, res_stream_pool], axis=3)\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower1 = down4\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower2 = down4\n",
    "    down4 = concatenate([tower1, tower2], axis=3)\n",
    "    down4 = Conv2D(24, (1, 1), padding='same')(down4)\n",
    "    down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)\n",
    "\n",
    "    res4 = UpSampling2D((32,32))(down4)\n",
    "    res_stream = Add()([res4, res_stream])  #640\n",
    "\n",
    "\n",
    "    # 10x15\n",
    "    res_stream_pool = MaxPooling2D((64, 64), strides=(64, 64))(res_stream)\n",
    "\n",
    "    center = concatenate([down4_pool, res_stream_pool], axis=3)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(24, (1, 1), padding='same')(center)\n",
    "\n",
    "    resCen = UpSampling2D((64,64))(center)\n",
    "    res_stream = Add()([resCen, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    up4 = UpSampling2D((2, 2))(center)\n",
    "    up4 = concatenate([res_stream_pool, up4], axis=3)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower1 = up4\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower2 = up4\n",
    "    up4 = concatenate([tower1,tower2], axis=3)\n",
    "    up4 = Conv2D(24, (1, 1), padding='same')(up4)\n",
    "\n",
    "    res_up4 = UpSampling2D((32, 32))(up4)\n",
    "    res_stream = Add()([res_up4, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "\n",
    "    up3 = UpSampling2D((2, 2))(up4)\n",
    "    up3 = concatenate([res_stream_pool, up3], axis=3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower1 = up3\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower2 = up3\n",
    "    up3 = concatenate([tower1, tower2], axis=3)\n",
    "    up3 = Conv2D(24, (1, 1), padding='same')(up3)\n",
    "\n",
    "    res_up3 = UpSampling2D((16, 16))(up3)\n",
    "    res_stream = Add()([res_up3, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool = MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    up2 = UpSampling2D((2, 2))(up3)\n",
    "    up2 = concatenate([res_stream_pool, up2], axis=3)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower1 = up2\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower2 = up2\n",
    "    up2 = concatenate([tower1,tower2], axis=3)\n",
    "    up2 = Conv2D(24, (1, 1), padding='same')(up2)\n",
    "\n",
    "    res_up2 = UpSampling2D((8, 8))(up2)\n",
    "    res_stream = Add()([res_up2, res_stream])  # 640\n",
    "\n",
    "    # 160x240\n",
    "    res_stream_pool = MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    up1 = UpSampling2D((2, 2))(up2)\n",
    "    up1 = concatenate([res_stream_pool, up1], axis=3)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower1 = up1\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower2 = up1\n",
    "    up1 = concatenate([tower1,tower2],axis=3)\n",
    "    up1 = Conv2D(24, (1, 1), padding='same')(up1)\n",
    "\n",
    "    res_up1 = UpSampling2D((4, 4))(up1)\n",
    "    res_stream = Add()([res_up1, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    res_stream_pool = MaxPooling2D((2, 2), strides=(2, 2))(res_stream)\n",
    "\n",
    "    up0 = UpSampling2D((2, 2))(up1)\n",
    "    up0 = concatenate([res_stream_pool, up0, down0], axis=3)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower1 = up0\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower2 = up0\n",
    "    up0 = concatenate([tower1,tower2],axis=3)\n",
    "    up0 = Conv2D(24, (1, 1), padding='same')(up0)\n",
    "\n",
    "    res_up0 = UpSampling2D((2, 2))(up0)\n",
    "    res_stream = Add()([res_up0, res_stream])  # 640\n",
    "\n",
    "    # 640x960\n",
    "\n",
    "    up0a = UpSampling2D((2, 2))(up0)\n",
    "    up0a = concatenate([res_stream, down0a, up0a], axis=3)\n",
    "\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower1 = up0a\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower2 = up0a\n",
    "    up0a = concatenate([tower1,tower2],axis=3)\n",
    "    up0a = Conv2D(24, (1, 1), padding='same')(up0a)\n",
    "    #up0a = SpatialDropout2D(0.5)(up0a)\n",
    "\n",
    "    res_stream = Add()([up0a, res_stream])  # 640\n",
    "    #keras.layers.convolutional.Conv2D(filters, kernel_size, strides=(1, 1), padding='valid', data_format=None,\n",
    "    #                                  dilation_rate=(1, 1), activation=None, use_bias=True,\n",
    "    #                                  kernel_initializer='glorot_uniform', bias_initializer='zeros',\n",
    "    #                                  kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None,\n",
    "    #                                  kernel_constraint=None, bias_constraint=None)\n",
    "\n",
    "    features = Conv2D(24, (3, 3), padding='same', use_bias=False)(res_stream)\n",
    "    classify = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True)(features)\n",
    "\n",
    "    #8 net experiment\n",
    "    #features2 = Conv2D(8, (5, 5), padding='same', use_bias=True, name = 'featurs2')(res_stream)\n",
    "    #classify2 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify2')(features2)\n",
    "    #features3 = Conv2D(8, (7, 7), padding='same', use_bias=True, name = 'featurs3')(res_stream)\n",
    "    #classify3 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify3')(features3)\n",
    "\n",
    "    #main_feature = Average()([classify1,classify2,classify3])\n",
    "    #main_output = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='main_output')(main_feature)\n",
    "\n",
    "    ##crf = CRF( units=128,sparse_target=True, activation='sigmoid',name = 'crf')(classify) # for word embedding. one dimentional\n",
    "\n",
    "    #output = CrfRnnLayer(image_dims=(640, 960),\n",
    "    #                     num_classes=1,\n",
    "    #                     theta_alpha=160.,\n",
    "    #                     theta_beta=3.,\n",
    "    #                     theta_gamma=3.,\n",
    "    #                     num_iterations=5,\n",
    "    #                     name='crfrnn')([classify, inputs])\n",
    "\n",
    "    #model = Model(inputs=inputs, outputs=[output,classify])\n",
    "    model = Model(inputs=inputs, outputs=classify)\n",
    "    #model = Model(inputs=inputs, outputs=[classify,tower1,tower2,tower3])\n",
    "\n",
    "    for i, layer in enumerate(model.layers):\n",
    "        print(i, layer.name)\n",
    "\n",
    "    model.summary()\n",
    "\n",
    "    # model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss=bce_dice_loss, metrics=[dice_loss])\n",
    "    #model.compile(optimizer=Adam(lr=0.0001), loss=bce_dice_loss, metrics=[dice_coeff])\n",
    "\n",
    "    model.compile(optimizer=Adam(lr=0.0002),\n",
    "                  loss=bce_dice_loss,\n",
    "                  #loss={'main_output': bce_dice_loss, 'aux_output': 'binary_crossentropy'},\n",
    "                  metrics=[dice_coeff])\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5},\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5})\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_unet_1280x1920_8(input_shape=(1280, 1920, 3),\n",
    "                       num_classes=1):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Res block\n",
    "\n",
    "    input_conv = Conv2D(24, (3, 3), padding='same')(inputs)\n",
    "    #input_conv = BatchNormalization()(input_conv)\n",
    "    input_conv= Activation('relu')(input_conv)\n",
    "\n",
    "    # 640x960 -\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(inputs)\n",
    "    #down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower1 = down0a\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(down0a)\n",
    "    #down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower2 = down0a\n",
    "    down0a = concatenate([tower1, tower2], axis=3)\n",
    "    down0a = Conv2D(24, (1, 1), padding='same')(down0a)\n",
    "    down0a_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0a)\n",
    "\n",
    "    res_stream = Add()([down0a, input_conv]) #640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0a_pool)\n",
    "    #down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower1 = down0\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0)\n",
    "    #down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower2 = down0\n",
    "    down0 = concatenate([tower1, tower2], axis=3)\n",
    "    down0 = Conv2D(24, (1, 1), padding='same')(down0)\n",
    "    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)\n",
    "\n",
    "    res0 = UpSampling2D((2,2))(down0)\n",
    "    res_stream = Add()([res0, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 160x240\n",
    "\n",
    "    res_stream_pool =  MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    down1 = concatenate([down0_pool, res_stream_pool], axis=3)\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    #down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower1 = down1\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    #down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower2 = down1\n",
    "    down1 = concatenate([tower1, tower2], axis=3)\n",
    "    down1 = Conv2D(24, (1, 1), padding='same')(down1)\n",
    "    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)\n",
    "\n",
    "    res1 = UpSampling2D((4,4))(down1) #640\n",
    "    res_stream = Add()([res1, res_stream])\n",
    "\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool =  MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    down2 = concatenate([down1_pool, res_stream_pool], axis=3)\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    #down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower1 = down2\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    #down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower2 = down2\n",
    "    down2 = concatenate([tower1, tower2], axis=3)\n",
    "    down2 = Conv2D(24, (1, 1), padding='same')(down2)\n",
    "    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)\n",
    "\n",
    "    res2 = UpSampling2D((8,8))(down2)\n",
    "    res_stream = Add()([res2, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "    down3 = concatenate([down2_pool, res_stream_pool], axis=3)\n",
    "\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    #down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower1 = down3\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    #down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower2 = down3\n",
    "    down3 = concatenate([tower1, tower2], axis=3)\n",
    "    down3 = Conv2D(24, (1, 1), padding='same')(down3)\n",
    "    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)\n",
    "\n",
    "    res3 = UpSampling2D((16,16))(down3)\n",
    "    res_stream = Add()([res3, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    down4 = concatenate([down3_pool, res_stream_pool], axis=3)\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    #down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower1 = down4\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    #down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower2 = down4\n",
    "    down4 = concatenate([tower1, tower2], axis=3)\n",
    "    down4 = Conv2D(24, (1, 1), padding='same')(down4)\n",
    "    down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)\n",
    "\n",
    "    res4 = UpSampling2D((32,32))(down4)\n",
    "    res_stream = Add()([res4, res_stream])  #640\n",
    "\n",
    "\n",
    "    # 10x15\n",
    "    res_stream_pool = MaxPooling2D((64, 64), strides=(64, 64))(res_stream)\n",
    "\n",
    "    center = concatenate([down4_pool, res_stream_pool], axis=3)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    #center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    #center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(24, (1, 1), padding='same')(center)\n",
    "\n",
    "    resCen = UpSampling2D((64,64))(center)\n",
    "    res_stream = Add()([resCen, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    up4 = UpSampling2D((2, 2))(center)\n",
    "    up4 = concatenate([res_stream_pool, up4], axis=3)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    #up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower1 = up4\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    #up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower2 = up4\n",
    "    up4 = concatenate([tower1,tower2], axis=3)\n",
    "    up4 = Conv2D(24, (1, 1), padding='same')(up4)\n",
    "\n",
    "    res_up4 = UpSampling2D((32, 32))(up4)\n",
    "    res_stream = Add()([res_up4, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "\n",
    "    up3 = UpSampling2D((2, 2))(up4)\n",
    "    up3 = concatenate([res_stream_pool, up3], axis=3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    #up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower1 = up3\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    #up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower2 = up3\n",
    "    up3 = concatenate([tower1, tower2], axis=3)\n",
    "    up3 = Conv2D(24, (1, 1), padding='same')(up3)\n",
    "\n",
    "    res_up3 = UpSampling2D((16, 16))(up3)\n",
    "    res_stream = Add()([res_up3, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool = MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    up2 = UpSampling2D((2, 2))(up3)\n",
    "    up2 = concatenate([res_stream_pool, up2], axis=3)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    #up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower1 = up2\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    #up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower2 = up2\n",
    "    up2 = concatenate([tower1,tower2], axis=3)\n",
    "    up2 = Conv2D(24, (1, 1), padding='same')(up2)\n",
    "\n",
    "    res_up2 = UpSampling2D((8, 8))(up2)\n",
    "    res_stream = Add()([res_up2, res_stream])  # 640\n",
    "\n",
    "    # 160x240\n",
    "    res_stream_pool = MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    up1 = UpSampling2D((2, 2))(up2)\n",
    "    up1 = concatenate([res_stream_pool, up1], axis=3)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    #up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower1 = up1\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    #up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower2 = up1\n",
    "    up1 = concatenate([tower1,tower2],axis=3)\n",
    "    up1 = Conv2D(24, (1, 1), padding='same')(up1)\n",
    "\n",
    "    res_up1 = UpSampling2D((4, 4))(up1)\n",
    "    res_stream = Add()([res_up1, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    res_stream_pool = MaxPooling2D((2, 2), strides=(2, 2))(res_stream)\n",
    "\n",
    "    up0 = UpSampling2D((2, 2))(up1)\n",
    "    up0 = concatenate([res_stream_pool, up0, down0], axis=3)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    #up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower1 = up0\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    #up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower2 = up0\n",
    "    up0 = concatenate([tower1,tower2],axis=3)\n",
    "    up0 = Conv2D(24, (1, 1), padding='same')(up0)\n",
    "\n",
    "    res_up0 = UpSampling2D((2, 2))(up0)\n",
    "    res_stream = Add()([res_up0, res_stream])  # 640\n",
    "\n",
    "    # 640x960\n",
    "\n",
    "    up0a = UpSampling2D((2, 2))(up0)\n",
    "    up0a = concatenate([res_stream, down0a, up0a], axis=3)\n",
    "\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    #up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower1 = up0a\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    #up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower2 = up0a\n",
    "    up0a = concatenate([tower1,tower2],axis=3)\n",
    "    up0a = Conv2D(24, (1, 1), padding='same')(up0a)\n",
    "    #up0a = SpatialDropout2D(0.5)(up0a)\n",
    "\n",
    "    res_stream = Add()([up0a, res_stream])  # 640\n",
    "\n",
    "    features = Conv2D(24, (3, 3), padding='same', use_bias=False)(res_stream)\n",
    "    classify = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True)(features)\n",
    "\n",
    "    #8 net experiment\n",
    "    #features2 = Conv2D(8, (5, 5), padding='same', use_bias=True, name = 'featurs2')(res_stream)\n",
    "    #classify2 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify2')(features2)\n",
    "    #features3 = Conv2D(8, (7, 7), padding='same', use_bias=True, name = 'featurs3')(res_stream)\n",
    "    #classify3 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify3')(features3)\n",
    "\n",
    "    #main_feature = Average()([classify1,classify2,classify3])\n",
    "    #main_output = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='main_output')(main_feature)\n",
    "\n",
    "    ##crf = CRF( units=128,sparse_target=True, activation='sigmoid',name = 'crf')(classify) # for word embedding. one dimentional\n",
    "\n",
    "    #output = CrfRnnLayer(image_dims=(640, 960),\n",
    "    #                     num_classes=1,\n",
    "    #                     theta_alpha=160.,\n",
    "    #                     theta_beta=3.,\n",
    "    #                     theta_gamma=3.,\n",
    "    #                     num_iterations=5,\n",
    "    #                     name='crfrnn')([classify, inputs])\n",
    "\n",
    "    #model = Model(inputs=inputs, outputs=[output,classify])\n",
    "    model = Model(inputs=inputs, outputs=classify)\n",
    "    #model = Model(inputs=inputs, outputs=[classify,tower1,tower2,tower3])\n",
    "\n",
    "    for i, layer in enumerate(model.layers):\n",
    "        print(i, layer.name)\n",
    "\n",
    "    model.summary()\n",
    "\n",
    "    # model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss=bce_dice_loss, metrics=[dice_loss])\n",
    "    #model.compile(optimizer=Adam(lr=0.0001), loss=bce_dice_loss, metrics=[dice_coeff])\n",
    "\n",
    "    model.compile(optimizer=Adam(lr=0.0002),\n",
    "                  loss=bce_dice_loss,\n",
    "                  #loss={'main_output': bce_dice_loss, 'aux_output': 'binary_crossentropy'},\n",
    "                  metrics=[dice_coeff])\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5},\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5})\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_unet_1024x1536_8(input_shape=(1024, 1536, 3),\n",
    "                       num_classes=1):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Res block\n",
    "\n",
    "    input_conv = Conv2D(24, (3, 3), padding='same')(inputs)\n",
    "    input_conv = BatchNormalization()(input_conv)\n",
    "    input_conv= Activation('relu')(input_conv)\n",
    "\n",
    "    # 640x960 -\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(inputs)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower1 = down0a\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(down0a)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower2 = down0a\n",
    "    down0a = concatenate([tower1, tower2], axis=3)\n",
    "    down0a = Conv2D(24, (1, 1), padding='same')(down0a)\n",
    "    down0a_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0a)\n",
    "\n",
    "    res_stream = Add()([down0a, input_conv]) #640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0a_pool)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower1 = down0\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower2 = down0\n",
    "    down0 = concatenate([tower1, tower2], axis=3)\n",
    "    down0 = Conv2D(24, (1, 1), padding='same')(down0)\n",
    "    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)\n",
    "\n",
    "    res0 = UpSampling2D((2,2))(down0)\n",
    "    res_stream = Add()([res0, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 160x240\n",
    "\n",
    "    res_stream_pool =  MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    down1 = concatenate([down0_pool, res_stream_pool], axis=3)\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower1 = down1\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower2 = down1\n",
    "    down1 = concatenate([tower1, tower2], axis=3)\n",
    "    down1 = Conv2D(24, (1, 1), padding='same')(down1)\n",
    "    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)\n",
    "\n",
    "    res1 = UpSampling2D((4,4))(down1) #640\n",
    "    res_stream = Add()([res1, res_stream])\n",
    "\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool =  MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    down2 = concatenate([down1_pool, res_stream_pool], axis=3)\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower1 = down2\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower2 = down2\n",
    "    down2 = concatenate([tower1, tower2], axis=3)\n",
    "    down2 = Conv2D(24, (1, 1), padding='same')(down2)\n",
    "    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)\n",
    "\n",
    "    res2 = UpSampling2D((8,8))(down2)\n",
    "    res_stream = Add()([res2, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "    down3 = concatenate([down2_pool, res_stream_pool], axis=3)\n",
    "\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower1 = down3\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower2 = down3\n",
    "    down3 = concatenate([tower1, tower2], axis=3)\n",
    "    down3 = Conv2D(24, (1, 1), padding='same')(down3)\n",
    "    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)\n",
    "\n",
    "    res3 = UpSampling2D((16,16))(down3)\n",
    "    res_stream = Add()([res3, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    down4 = concatenate([down3_pool, res_stream_pool], axis=3)\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower1 = down4\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower2 = down4\n",
    "    down4 = concatenate([tower1, tower2], axis=3)\n",
    "    down4 = Conv2D(24, (1, 1), padding='same')(down4)\n",
    "    down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)\n",
    "\n",
    "    res4 = UpSampling2D((32,32))(down4)\n",
    "    res_stream = Add()([res4, res_stream])  #640\n",
    "\n",
    "\n",
    "    # 10x15\n",
    "    res_stream_pool = MaxPooling2D((64, 64), strides=(64, 64))(res_stream)\n",
    "\n",
    "    center = concatenate([down4_pool, res_stream_pool], axis=3)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(24, (1, 1), padding='same')(center)\n",
    "\n",
    "    resCen = UpSampling2D((64,64))(center)\n",
    "    res_stream = Add()([resCen, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    up4 = UpSampling2D((2, 2))(center)\n",
    "    up4 = concatenate([res_stream_pool, up4], axis=3)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower1 = up4\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower2 = up4\n",
    "    up4 = concatenate([tower1,tower2], axis=3)\n",
    "    up4 = Conv2D(24, (1, 1), padding='same')(up4)\n",
    "\n",
    "    res_up4 = UpSampling2D((32, 32))(up4)\n",
    "    res_stream = Add()([res_up4, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "\n",
    "    up3 = UpSampling2D((2, 2))(up4)\n",
    "    up3 = concatenate([res_stream_pool, up3], axis=3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower1 = up3\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower2 = up3\n",
    "    up3 = concatenate([tower1, tower2], axis=3)\n",
    "    up3 = Conv2D(24, (1, 1), padding='same')(up3)\n",
    "\n",
    "    res_up3 = UpSampling2D((16, 16))(up3)\n",
    "    res_stream = Add()([res_up3, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool = MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    up2 = UpSampling2D((2, 2))(up3)\n",
    "    up2 = concatenate([res_stream_pool, up2], axis=3)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower1 = up2\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower2 = up2\n",
    "    up2 = concatenate([tower1,tower2], axis=3)\n",
    "    up2 = Conv2D(24, (1, 1), padding='same')(up2)\n",
    "\n",
    "    res_up2 = UpSampling2D((8, 8))(up2)\n",
    "    res_stream = Add()([res_up2, res_stream])  # 640\n",
    "\n",
    "    # 160x240\n",
    "    res_stream_pool = MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    up1 = UpSampling2D((2, 2))(up2)\n",
    "    up1 = concatenate([res_stream_pool, up1], axis=3)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower1 = up1\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower2 = up1\n",
    "    up1 = concatenate([tower1,tower2],axis=3)\n",
    "    up1 = Conv2D(24, (1, 1), padding='same')(up1)\n",
    "\n",
    "    res_up1 = UpSampling2D((4, 4))(up1)\n",
    "    res_stream = Add()([res_up1, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    res_stream_pool = MaxPooling2D((2, 2), strides=(2, 2))(res_stream)\n",
    "\n",
    "    up0 = UpSampling2D((2, 2))(up1)\n",
    "    up0 = concatenate([res_stream_pool, up0, down0], axis=3)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower1 = up0\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower2 = up0\n",
    "    up0 = concatenate([tower1,tower2],axis=3)\n",
    "    up0 = Conv2D(24, (1, 1), padding='same')(up0)\n",
    "\n",
    "    res_up0 = UpSampling2D((2, 2))(up0)\n",
    "    res_stream = Add()([res_up0, res_stream])  # 640\n",
    "\n",
    "    # 640x960\n",
    "\n",
    "    up0a = UpSampling2D((2, 2))(up0)\n",
    "    up0a = concatenate([res_stream, down0a, up0a], axis=3)\n",
    "\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower1 = up0a\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower2 = up0a\n",
    "    up0a = concatenate([tower1,tower2],axis=3)\n",
    "    up0a = Conv2D(24, (1, 1), padding='same')(up0a)\n",
    "    #up0a = SpatialDropout2D(0.5)(up0a)\n",
    "\n",
    "    res_stream = Add()([up0a, res_stream])  # 640\n",
    "    #keras.layers.convolutional.Conv2D(filters, kernel_size, strides=(1, 1), padding='valid', data_format=None,\n",
    "    #                                  dilation_rate=(1, 1), activation=None, use_bias=True,\n",
    "    #                                  kernel_initializer='glorot_uniform', bias_initializer='zeros',\n",
    "    #                                  kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None,\n",
    "    #                                  kernel_constraint=None, bias_constraint=None)\n",
    "\n",
    "    features = Conv2D(24, (3, 3), padding='same', use_bias=False)(res_stream)\n",
    "    classify = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True)(features)\n",
    "\n",
    "    #8 net experiment\n",
    "    #features2 = Conv2D(8, (5, 5), padding='same', use_bias=True, name = 'featurs2')(res_stream)\n",
    "    #classify2 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify2')(features2)\n",
    "    #features3 = Conv2D(8, (7, 7), padding='same', use_bias=True, name = 'featurs3')(res_stream)\n",
    "    #classify3 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify3')(features3)\n",
    "\n",
    "    #main_feature = Average()([classify1,classify2,classify3])\n",
    "    #main_output = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='main_output')(main_feature)\n",
    "\n",
    "    ##crf = CRF( units=128,sparse_target=True, activation='sigmoid',name = 'crf')(classify) # for word embedding. one dimentional\n",
    "\n",
    "    #output = CrfRnnLayer(image_dims=(640, 960),\n",
    "    #                     num_classes=1,\n",
    "    #                     theta_alpha=160.,\n",
    "    #                     theta_beta=3.,\n",
    "    #                     theta_gamma=3.,\n",
    "    #                     num_iterations=5,\n",
    "    #                     name='crfrnn')([classify, inputs])\n",
    "\n",
    "    #model = Model(inputs=inputs, outputs=[output,classify])\n",
    "    model = Model(inputs=inputs, outputs=classify)\n",
    "    #model = Model(inputs=inputs, outputs=[classify,tower1,tower2,tower3])\n",
    "\n",
    "    for i, layer in enumerate(model.layers):\n",
    "        print(i, layer.name)\n",
    "\n",
    "    model.summary()\n",
    "\n",
    "    # model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss=bce_dice_loss, metrics=[dice_loss])\n",
    "    #model.compile(optimizer=Adam(lr=0.0001), loss=bce_dice_loss, metrics=[dice_coeff])\n",
    "\n",
    "    model.compile(optimizer=Adam(lr=0.00015),\n",
    "                  loss=bce_dice_loss,\n",
    "                  #loss={'main_output': bce_dice_loss, 'aux_output': 'binary_crossentropy'},\n",
    "                  metrics=[dice_coeff])\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5},\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5})\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_unet_1280x1280_8(input_shape=(1280, 1280, 3),\n",
    "                       num_classes=1):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Res block\n",
    "\n",
    "    input_conv = Conv2D(24, (3, 3), padding='same')(inputs)\n",
    "    input_conv = BatchNormalization()(input_conv)\n",
    "    input_conv= Activation('relu')(input_conv)\n",
    "\n",
    "    # 640x960 -\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(inputs)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower1 = down0a\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(down0a)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower2 = down0a\n",
    "    down0a = concatenate([tower1, tower2], axis=3)\n",
    "    down0a = Conv2D(24, (1, 1), padding='same')(down0a)\n",
    "    down0a_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0a)\n",
    "\n",
    "    res_stream = Add()([down0a, input_conv]) #640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0a_pool)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower1 = down0\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower2 = down0\n",
    "    down0 = concatenate([tower1, tower2], axis=3)\n",
    "    down0 = Conv2D(24, (1, 1), padding='same')(down0)\n",
    "    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)\n",
    "\n",
    "    res0 = UpSampling2D((2,2))(down0)\n",
    "    res_stream = Add()([res0, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 160x240\n",
    "\n",
    "    res_stream_pool =  MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    down1 = concatenate([down0_pool, res_stream_pool], axis=3)\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower1 = down1\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower2 = down1\n",
    "    down1 = concatenate([tower1, tower2], axis=3)\n",
    "    down1 = Conv2D(24, (1, 1), padding='same')(down1)\n",
    "    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)\n",
    "\n",
    "    res1 = UpSampling2D((4,4))(down1) #640\n",
    "    res_stream = Add()([res1, res_stream])\n",
    "\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool =  MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    down2 = concatenate([down1_pool, res_stream_pool], axis=3)\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower1 = down2\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower2 = down2\n",
    "    down2 = concatenate([tower1, tower2], axis=3)\n",
    "    down2 = Conv2D(24, (1, 1), padding='same')(down2)\n",
    "    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)\n",
    "\n",
    "    res2 = UpSampling2D((8,8))(down2)\n",
    "    res_stream = Add()([res2, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "    down3 = concatenate([down2_pool, res_stream_pool], axis=3)\n",
    "\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower1 = down3\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower2 = down3\n",
    "    down3 = concatenate([tower1, tower2], axis=3)\n",
    "    down3 = Conv2D(24, (1, 1), padding='same')(down3)\n",
    "    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)\n",
    "\n",
    "    res3 = UpSampling2D((16,16))(down3)\n",
    "    res_stream = Add()([res3, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    down4 = concatenate([down3_pool, res_stream_pool], axis=3)\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower1 = down4\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower2 = down4\n",
    "    down4 = concatenate([tower1, tower2], axis=3)\n",
    "    down4 = Conv2D(24, (1, 1), padding='same')(down4)\n",
    "    down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)\n",
    "\n",
    "    res4 = UpSampling2D((32,32))(down4)\n",
    "    res_stream = Add()([res4, res_stream])  #640\n",
    "\n",
    "\n",
    "    # 10x15\n",
    "    res_stream_pool = MaxPooling2D((64, 64), strides=(64, 64))(res_stream)\n",
    "\n",
    "    center = concatenate([down4_pool, res_stream_pool], axis=3)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(24, (1, 1), padding='same')(center)\n",
    "\n",
    "    resCen = UpSampling2D((64,64))(center)\n",
    "    res_stream = Add()([resCen, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    up4 = UpSampling2D((2, 2))(center)\n",
    "    up4 = concatenate([res_stream_pool, up4], axis=3)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower1 = up4\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower2 = up4\n",
    "    up4 = concatenate([tower1,tower2], axis=3)\n",
    "    up4 = Conv2D(24, (1, 1), padding='same')(up4)\n",
    "\n",
    "    res_up4 = UpSampling2D((32, 32))(up4)\n",
    "    res_stream = Add()([res_up4, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "\n",
    "    up3 = UpSampling2D((2, 2))(up4)\n",
    "    up3 = concatenate([res_stream_pool, up3], axis=3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower1 = up3\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower2 = up3\n",
    "    up3 = concatenate([tower1, tower2], axis=3)\n",
    "    up3 = Conv2D(24, (1, 1), padding='same')(up3)\n",
    "\n",
    "    res_up3 = UpSampling2D((16, 16))(up3)\n",
    "    res_stream = Add()([res_up3, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool = MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    up2 = UpSampling2D((2, 2))(up3)\n",
    "    up2 = concatenate([res_stream_pool, up2], axis=3)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower1 = up2\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower2 = up2\n",
    "    up2 = concatenate([tower1,tower2], axis=3)\n",
    "    up2 = Conv2D(24, (1, 1), padding='same')(up2)\n",
    "\n",
    "    res_up2 = UpSampling2D((8, 8))(up2)\n",
    "    res_stream = Add()([res_up2, res_stream])  # 640\n",
    "\n",
    "    # 160x240\n",
    "    res_stream_pool = MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    up1 = UpSampling2D((2, 2))(up2)\n",
    "    up1 = concatenate([res_stream_pool, up1], axis=3)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower1 = up1\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower2 = up1\n",
    "    up1 = concatenate([tower1,tower2],axis=3)\n",
    "    up1 = Conv2D(24, (1, 1), padding='same')(up1)\n",
    "\n",
    "    res_up1 = UpSampling2D((4, 4))(up1)\n",
    "    res_stream = Add()([res_up1, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    res_stream_pool = MaxPooling2D((2, 2), strides=(2, 2))(res_stream)\n",
    "\n",
    "    up0 = UpSampling2D((2, 2))(up1)\n",
    "    up0 = concatenate([res_stream_pool, up0, down0], axis=3)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower1 = up0\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower2 = up0\n",
    "    up0 = concatenate([tower1,tower2],axis=3)\n",
    "    up0 = Conv2D(24, (1, 1), padding='same')(up0)\n",
    "\n",
    "    res_up0 = UpSampling2D((2, 2))(up0)\n",
    "    res_stream = Add()([res_up0, res_stream])  # 640\n",
    "\n",
    "    # 640x960\n",
    "\n",
    "    up0a = UpSampling2D((2, 2))(up0)\n",
    "    up0a = concatenate([res_stream, down0a, up0a], axis=3)\n",
    "\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower1 = up0a\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower2 = up0a\n",
    "    up0a = concatenate([tower1,tower2],axis=3)\n",
    "    up0a = Conv2D(24, (1, 1), padding='same')(up0a)\n",
    "    #up0a = SpatialDropout2D(0.5)(up0a)\n",
    "\n",
    "    res_stream = Add()([up0a, res_stream])  # 640\n",
    "    #keras.layers.convolutional.Conv2D(filters, kernel_size, strides=(1, 1), padding='valid', data_format=None,\n",
    "    #                                  dilation_rate=(1, 1), activation=None, use_bias=True,\n",
    "    #                                  kernel_initializer='glorot_uniform', bias_initializer='zeros',\n",
    "    #                                  kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None,\n",
    "    #                                  kernel_constraint=None, bias_constraint=None)\n",
    "\n",
    "    features = Conv2D(24, (3, 3), padding='same', use_bias=False)(res_stream)\n",
    "    classify = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True)(features)\n",
    "\n",
    "    #8 net experiment\n",
    "    #features2 = Conv2D(8, (5, 5), padding='same', use_bias=True, name = 'featurs2')(res_stream)\n",
    "    #classify2 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify2')(features2)\n",
    "    #features3 = Conv2D(8, (7, 7), padding='same', use_bias=True, name = 'featurs3')(res_stream)\n",
    "    #classify3 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify3')(features3)\n",
    "\n",
    "    #main_feature = Average()([classify1,classify2,classify3])\n",
    "    #main_output = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='main_output')(main_feature)\n",
    "\n",
    "    ##crf = CRF( units=128,sparse_target=True, activation='sigmoid',name = 'crf')(classify) # for word embedding. one dimentional\n",
    "\n",
    "    #output = CrfRnnLayer(image_dims=(640, 960),\n",
    "    #                     num_classes=1,\n",
    "    #                     theta_alpha=160.,\n",
    "    #                     theta_beta=3.,\n",
    "    #                     theta_gamma=3.,\n",
    "    #                     num_iterations=5,\n",
    "    #                     name='crfrnn')([classify, inputs])\n",
    "\n",
    "    #model = Model(inputs=inputs, outputs=[output,classify])\n",
    "    model = Model(inputs=inputs, outputs=classify)\n",
    "    #model = Model(inputs=inputs, outputs=[classify,tower1,tower2,tower3])\n",
    "\n",
    "    for i, layer in enumerate(model.layers):\n",
    "        print(i, layer.name)\n",
    "\n",
    "    model.summary()\n",
    "\n",
    "    # model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss=bce_dice_loss, metrics=[dice_loss])\n",
    "    #model.compile(optimizer=Adam(lr=0.0001), loss=bce_dice_loss, metrics=[dice_coeff])\n",
    "\n",
    "    model.compile(optimizer=Adam(lr=0.0001),\n",
    "                  loss=bce_dice_loss,\n",
    "                  #loss={'main_output': bce_dice_loss, 'aux_output': 'binary_crossentropy'},\n",
    "                  metrics=[dice_coeff])\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5},\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5})\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_unet_1280x960_8(input_shape=(1280, 960, 3),\n",
    "                       num_classes=1):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Res block\n",
    "\n",
    "    input_conv = Conv2D(24, (3, 3), padding='same')(inputs)\n",
    "    input_conv = BatchNormalization()(input_conv)\n",
    "    input_conv= Activation('relu')(input_conv)\n",
    "\n",
    "    # 640x960 -\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(inputs)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower1 = down0a\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(down0a)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower2 = down0a\n",
    "    down0a = concatenate([tower1, tower2], axis=3)\n",
    "    down0a = Conv2D(24, (1, 1), padding='same')(down0a)\n",
    "    down0a_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0a)\n",
    "\n",
    "    res_stream = Add()([down0a, input_conv]) #640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0a_pool)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower1 = down0\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower2 = down0\n",
    "    down0 = concatenate([tower1, tower2], axis=3)\n",
    "    down0 = Conv2D(24, (1, 1), padding='same')(down0)\n",
    "    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)\n",
    "\n",
    "    res0 = UpSampling2D((2,2))(down0)\n",
    "    res_stream = Add()([res0, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 160x240\n",
    "\n",
    "    res_stream_pool =  MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    down1 = concatenate([down0_pool, res_stream_pool], axis=3)\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower1 = down1\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower2 = down1\n",
    "    down1 = concatenate([tower1, tower2], axis=3)\n",
    "    down1 = Conv2D(24, (1, 1), padding='same')(down1)\n",
    "    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)\n",
    "\n",
    "    res1 = UpSampling2D((4,4))(down1) #640\n",
    "    res_stream = Add()([res1, res_stream])\n",
    "\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool =  MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    down2 = concatenate([down1_pool, res_stream_pool], axis=3)\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower1 = down2\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower2 = down2\n",
    "    down2 = concatenate([tower1, tower2], axis=3)\n",
    "    down2 = Conv2D(24, (1, 1), padding='same')(down2)\n",
    "    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)\n",
    "\n",
    "    res2 = UpSampling2D((8,8))(down2)\n",
    "    res_stream = Add()([res2, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "    down3 = concatenate([down2_pool, res_stream_pool], axis=3)\n",
    "\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower1 = down3\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower2 = down3\n",
    "    down3 = concatenate([tower1, tower2], axis=3)\n",
    "    down3 = Conv2D(24, (1, 1), padding='same')(down3)\n",
    "    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)\n",
    "\n",
    "    res3 = UpSampling2D((16,16))(down3)\n",
    "    res_stream = Add()([res3, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    down4 = concatenate([down3_pool, res_stream_pool], axis=3)\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower1 = down4\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower2 = down4\n",
    "    down4 = concatenate([tower1, tower2], axis=3)\n",
    "    down4 = Conv2D(24, (1, 1), padding='same')(down4)\n",
    "    down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)\n",
    "\n",
    "    res4 = UpSampling2D((32,32))(down4)\n",
    "    res_stream = Add()([res4, res_stream])  #640\n",
    "\n",
    "\n",
    "    # 10x15\n",
    "    res_stream_pool = MaxPooling2D((64, 64), strides=(64, 64))(res_stream)\n",
    "\n",
    "    center = concatenate([down4_pool, res_stream_pool], axis=3)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(24, (1, 1), padding='same')(center)\n",
    "\n",
    "    resCen = UpSampling2D((64,64))(center)\n",
    "    res_stream = Add()([resCen, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    up4 = UpSampling2D((2, 2))(center)\n",
    "    up4 = concatenate([res_stream_pool, up4], axis=3)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower1 = up4\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower2 = up4\n",
    "    up4 = concatenate([tower1,tower2], axis=3)\n",
    "    up4 = Conv2D(24, (1, 1), padding='same')(up4)\n",
    "\n",
    "    res_up4 = UpSampling2D((32, 32))(up4)\n",
    "    res_stream = Add()([res_up4, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "\n",
    "    up3 = UpSampling2D((2, 2))(up4)\n",
    "    up3 = concatenate([res_stream_pool, up3], axis=3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower1 = up3\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower2 = up3\n",
    "    up3 = concatenate([tower1, tower2], axis=3)\n",
    "    up3 = Conv2D(24, (1, 1), padding='same')(up3)\n",
    "\n",
    "    res_up3 = UpSampling2D((16, 16))(up3)\n",
    "    res_stream = Add()([res_up3, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool = MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    up2 = UpSampling2D((2, 2))(up3)\n",
    "    up2 = concatenate([res_stream_pool, up2], axis=3)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower1 = up2\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower2 = up2\n",
    "    up2 = concatenate([tower1,tower2], axis=3)\n",
    "    up2 = Conv2D(24, (1, 1), padding='same')(up2)\n",
    "\n",
    "    res_up2 = UpSampling2D((8, 8))(up2)\n",
    "    res_stream = Add()([res_up2, res_stream])  # 640\n",
    "\n",
    "    # 160x240\n",
    "    res_stream_pool = MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    up1 = UpSampling2D((2, 2))(up2)\n",
    "    up1 = concatenate([res_stream_pool, up1], axis=3)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower1 = up1\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower2 = up1\n",
    "    up1 = concatenate([tower1,tower2],axis=3)\n",
    "    up1 = Conv2D(24, (1, 1), padding='same')(up1)\n",
    "\n",
    "    res_up1 = UpSampling2D((4, 4))(up1)\n",
    "    res_stream = Add()([res_up1, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    res_stream_pool = MaxPooling2D((2, 2), strides=(2, 2))(res_stream)\n",
    "\n",
    "    up0 = UpSampling2D((2, 2))(up1)\n",
    "    up0 = concatenate([res_stream_pool, up0, down0], axis=3)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower1 = up0\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower2 = up0\n",
    "    up0 = concatenate([tower1,tower2],axis=3)\n",
    "    up0 = Conv2D(24, (1, 1), padding='same')(up0)\n",
    "\n",
    "    res_up0 = UpSampling2D((2, 2))(up0)\n",
    "    res_stream = Add()([res_up0, res_stream])  # 640\n",
    "\n",
    "    # 640x960\n",
    "\n",
    "    up0a = UpSampling2D((2, 2))(up0)\n",
    "    up0a = concatenate([res_stream, down0a, up0a], axis=3)\n",
    "\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower1 = up0a\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower2 = up0a\n",
    "    up0a = concatenate([tower1,tower2],axis=3)\n",
    "    up0a = Conv2D(24, (1, 1), padding='same')(up0a)\n",
    "    #up0a = SpatialDropout2D(0.5)(up0a)\n",
    "\n",
    "    res_stream = Add()([up0a, res_stream])  # 640\n",
    "    #keras.layers.convolutional.Conv2D(filters, kernel_size, strides=(1, 1), padding='valid', data_format=None,\n",
    "    #                                  dilation_rate=(1, 1), activation=None, use_bias=True,\n",
    "    #                                  kernel_initializer='glorot_uniform', bias_initializer='zeros',\n",
    "    #                                  kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None,\n",
    "    #                                  kernel_constraint=None, bias_constraint=None)\n",
    "\n",
    "    features = Conv2D(24, (3, 3), padding='same', use_bias=False)(res_stream)\n",
    "    classify = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True)(features)\n",
    "\n",
    "    #8 net experiment\n",
    "    #features2 = Conv2D(8, (5, 5), padding='same', use_bias=True, name = 'featurs2')(res_stream)\n",
    "    #classify2 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify2')(features2)\n",
    "    #features3 = Conv2D(8, (7, 7), padding='same', use_bias=True, name = 'featurs3')(res_stream)\n",
    "    #classify3 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify3')(features3)\n",
    "\n",
    "    #main_feature = Average()([classify1,classify2,classify3])\n",
    "    #main_output = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='main_output')(main_feature)\n",
    "\n",
    "    ##crf = CRF( units=128,sparse_target=True, activation='sigmoid',name = 'crf')(classify) # for word embedding. one dimentional\n",
    "\n",
    "    #output = CrfRnnLayer(image_dims=(640, 960),\n",
    "    #                     num_classes=1,\n",
    "    #                     theta_alpha=160.,\n",
    "    #                     theta_beta=3.,\n",
    "    #                     theta_gamma=3.,\n",
    "    #                     num_iterations=5,\n",
    "    #                     name='crfrnn')([classify, inputs])\n",
    "\n",
    "    #model = Model(inputs=inputs, outputs=[output,classify])\n",
    "    model = Model(inputs=inputs, outputs=classify)\n",
    "    #model = Model(inputs=inputs, outputs=[classify,tower1,tower2,tower3])\n",
    "\n",
    "    for i, layer in enumerate(model.layers):\n",
    "        print(i, layer.name)\n",
    "\n",
    "    model.summary()\n",
    "\n",
    "    # model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss=bce_dice_loss, metrics=[dice_loss])\n",
    "    #model.compile(optimizer=Adam(lr=0.0001), loss=bce_dice_loss, metrics=[dice_coeff])\n",
    "\n",
    "    model.compile(optimizer=Adam(lr=0.0001),\n",
    "                  loss=bce_dice_loss,\n",
    "                  #loss={'main_output': bce_dice_loss, 'aux_output': 'binary_crossentropy'},\n",
    "                  metrics=[dice_coeff])\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5},\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5})\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "##########################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_unet_640_960_11(input_shape=(640, 960, 3),\n",
    "                       num_classes=1):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "\n",
    "    input_conv = Conv2D(24, (3, 3), padding='same')(inputs)\n",
    "    input_conv = BatchNormalization()(input_conv)\n",
    "    input_conv= Activation('relu')(input_conv)\n",
    "\n",
    "    # 640x960 -\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(inputs)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower1 = down0a\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(down0a)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower2 = down0a\n",
    "    down0a = concatenate([tower1, tower2], axis=3)\n",
    "    down0a = Conv2D(24, (1, 1), padding='same')(down0a)\n",
    "    down0a_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0a)\n",
    "\n",
    "    res_stream = Add()([down0a, input_conv])  # 640\n",
    "\n",
    "    # 320x480\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0a_pool)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower1 = down0\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower2 = down0\n",
    "    down0 = concatenate([tower1, tower2], axis=3)\n",
    "    down0 = Conv2D(24, (1, 1), padding='same')(down0)\n",
    "    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)\n",
    "\n",
    "    res0 = UpSampling2D((2, 2))(down0)\n",
    "    res_stream = Add()([res0, res_stream])  # 640\n",
    "\n",
    "    # 160x240\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    down1 = concatenate([down0_pool, res_stream_pool], axis=3)\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower1 = down1\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower2 = down1\n",
    "    down1 = concatenate([tower1, tower2], axis=3)\n",
    "    down1 = Conv2D(24, (1, 1), padding='same')(down1)\n",
    "    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)\n",
    "\n",
    "    res1 = UpSampling2D((4, 4))(down1)  # 640\n",
    "    res_stream = Add()([res1, res_stream])\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool =  MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    down2 = concatenate([down1_pool, res_stream_pool], axis=3)\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower1 = down2\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower2 = down2\n",
    "    down2 = concatenate([tower1, tower2], axis=3)\n",
    "    down2 = Conv2D(24, (1, 1), padding='same')(down2)\n",
    "    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)\n",
    "\n",
    "    res2 = UpSampling2D((8,8))(down2)\n",
    "    res_stream = Add()([res2, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "    down3 = concatenate([down2_pool, res_stream_pool], axis=3)\n",
    "\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower1 = down3\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower2 = down3\n",
    "    down3 = concatenate([tower1, tower2], axis=3)\n",
    "    down3 = Conv2D(24, (1, 1), padding='same')(down3)\n",
    "    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)\n",
    "\n",
    "    res3 = UpSampling2D((16,16))(down3)\n",
    "    res_stream = Add()([res3, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    down4 = concatenate([down3_pool, res_stream_pool], axis=3)\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower1 = down4\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower2 = down4\n",
    "    down4 = concatenate([tower1, tower2], axis=3)\n",
    "    down4 = Conv2D(24, (1, 1), padding='same')(down4)\n",
    "    down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)\n",
    "\n",
    "    res4 = UpSampling2D((32,32))(down4)\n",
    "    res_stream = Add()([res4, res_stream])  #640\n",
    "\n",
    "\n",
    "    # 10x15\n",
    "    res_stream_pool = MaxPooling2D((64, 64), strides=(64, 64))(res_stream)\n",
    "\n",
    "    center = concatenate([down4_pool, res_stream_pool], axis=3)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(24, (1, 1), padding='same')(center)\n",
    "\n",
    "    resCen = UpSampling2D((64,64))(center)\n",
    "    res_stream = Add()([resCen, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    up4 = UpSampling2D((2, 2))(center)\n",
    "    up4 = concatenate([res_stream_pool, up4], axis=3)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower1 = up4\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower2 = up4\n",
    "    up4 = concatenate([tower1,tower2], axis=3)\n",
    "    up4 = Conv2D(24, (1, 1), padding='same')(up4)\n",
    "\n",
    "    res_up4 = UpSampling2D((32, 32))(up4)\n",
    "    res_stream = Add()([res_up4, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "\n",
    "    up3 = UpSampling2D((2, 2))(up4)\n",
    "    up3 = concatenate([res_stream_pool, up3], axis=3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower1 = up3\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower2 = up3\n",
    "    up3 = concatenate([tower1, tower2], axis=3)\n",
    "    up3 = Conv2D(24, (1, 1), padding='same')(up3)\n",
    "\n",
    "    res_up3 = UpSampling2D((16, 16))(up3)\n",
    "    res_stream = Add()([res_up3, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool = MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    up2 = UpSampling2D((2, 2))(up3)\n",
    "    up2 = concatenate([res_stream_pool, up2], axis=3)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower1 = up2\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower2 = up2\n",
    "    up2 = concatenate([tower1,tower2], axis=3)\n",
    "    up2 = Conv2D(24, (1, 1), padding='same')(up2)\n",
    "\n",
    "    res_up2 = UpSampling2D((8, 8))(up2)\n",
    "    res_stream = Add()([res_up2, res_stream])  # 640\n",
    "\n",
    "    # 160x240\n",
    "    res_stream_pool = MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    up1 = UpSampling2D((2, 2))(up2)\n",
    "    up1 = concatenate([res_stream_pool, up1], axis=3)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower1 = up1\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower2 = up1\n",
    "    up1 = concatenate([tower1,tower2],axis=3)\n",
    "    up1 = Conv2D(24, (1, 1), padding='same')(up1)\n",
    "\n",
    "    res_up1 = UpSampling2D((4, 4))(up1)\n",
    "    res_stream = Add()([res_up1, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    # 320x480\n",
    "    res_stream_pool = MaxPooling2D((2, 2), strides=(2, 2))(res_stream)\n",
    "\n",
    "    up0 = UpSampling2D((2, 2))(up1)\n",
    "    up0 = concatenate([res_stream_pool, up0, down0], axis=3)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower1 = up0\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower2 = up0\n",
    "    up0 = concatenate([tower1, tower2], axis=3)\n",
    "    up0 = Conv2D(24, (1, 1), padding='same')(up0)\n",
    "\n",
    "    res_up0 = UpSampling2D((2, 2))(up0)\n",
    "    res_stream = Add()([res_up0, res_stream])  # 640\n",
    "\n",
    "    # 640x960\n",
    "\n",
    "    up0a = UpSampling2D((2, 2))(up0)\n",
    "    up0a = concatenate([res_stream, down0a, up0a], axis=3)\n",
    "\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower1 = up0a\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower2 = up0a\n",
    "    up0a = concatenate([tower1, tower2], axis=3)\n",
    "    up0a = Conv2D(24, (1, 1), padding='same')(up0a)\n",
    "    # up0a = SpatialDropout2D(0.5)(up0a)\n",
    "\n",
    "    res_stream = Add()([up0a, res_stream])  # 640\n",
    "\n",
    "    feature_1 = Conv2D(16, (31, 1), activation='relu', padding='same', dilation_rate=(1, 1))(res_stream)\n",
    "    feature_1 = BatchNormalization()(feature_1)\n",
    "    feature_1 = Conv2D(16, (1, 31), activation='relu', padding='same', dilation_rate=(1, 1))(feature_1)\n",
    "    feature_1 = BatchNormalization()(feature_1)\n",
    "\n",
    "    feature_2 = Conv2D(16, (1, 31), activation='relu', padding='same', dilation_rate=(1, 1))(res_stream)\n",
    "    feature_2 = BatchNormalization()(feature_2)\n",
    "    feature_2 = Conv2D(16, (31, 1), activation='relu', padding='same', dilation_rate=(1, 1))(feature_2)\n",
    "    feature_2 = BatchNormalization()(feature_2)\n",
    "\n",
    "    feature = Add()([feature_1, feature_2])\n",
    "\n",
    "    main_feature = Conv2D(16, (3, 3), padding='same', use_bias=True, name = 'feature_main_cnv1')(feature)\n",
    "    main_feature = BatchNormalization()(main_feature)\n",
    "    main_feature = Activation('relu')(main_feature)\n",
    "    main_feature = Conv2D(16, (3, 3), padding='same', use_bias=True, name='feature_main_cnv2')(main_feature)\n",
    "\n",
    "    classify_main = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify_main')(main_feature)\n",
    "\n",
    "\n",
    "    #model = Model(inputs=inputs, outputs=[output,classify])\n",
    "    model = Model(inputs=inputs, outputs=classify_main)\n",
    "    #model = Model(inputs=inputs, outputs=[classify,tower1,tower2,tower3])\n",
    "\n",
    "    for i, layer in enumerate(model.layers):\n",
    "        print(i, layer.name)\n",
    "\n",
    "    model.summary()\n",
    "\n",
    "    model.compile(optimizer=Adam(lr=0.0005),\n",
    "                  loss=bce_dice_loss,\n",
    "                  #loss={'main_output': bce_dice_loss, 'aux_output': 'binary_crossentropy'},\n",
    "                  metrics=[dice_coeff])\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5},\n",
    "                  #loss_weights={'classify_main': 1., 'classify_edge': 0.2,  'classify_res': 0.2})\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_unet_832x1216_11(input_shape=(832, 1216, 3),\n",
    "                       num_classes=1):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "\n",
    "    input_conv = Conv2D(24, (3, 3), padding='same')(inputs)\n",
    "    input_conv = BatchNormalization()(input_conv)\n",
    "    input_conv= Activation('relu')(input_conv)\n",
    "\n",
    "    # 640x960 -\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(inputs)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower1 = down0a\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(down0a)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower2 = down0a\n",
    "    down0a = concatenate([tower1, tower2], axis=3)\n",
    "    down0a = Conv2D(24, (1, 1), padding='same')(down0a)\n",
    "    down0a_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0a)\n",
    "\n",
    "    res_stream = Add()([down0a, input_conv])  # 640\n",
    "\n",
    "    # 320x480\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0a_pool)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower1 = down0\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower2 = down0\n",
    "    down0 = concatenate([tower1, tower2], axis=3)\n",
    "    down0 = Conv2D(24, (1, 1), padding='same')(down0)\n",
    "    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)\n",
    "\n",
    "    res0 = UpSampling2D((2, 2))(down0)\n",
    "    res_stream = Add()([res0, res_stream])  # 640\n",
    "\n",
    "    # 160x240\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    down1 = concatenate([down0_pool, res_stream_pool], axis=3)\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower1 = down1\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower2 = down1\n",
    "    down1 = concatenate([tower1, tower2], axis=3)\n",
    "    down1 = Conv2D(24, (1, 1), padding='same')(down1)\n",
    "    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)\n",
    "\n",
    "    res1 = UpSampling2D((4, 4))(down1)  # 640\n",
    "    res_stream = Add()([res1, res_stream])\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool =  MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    down2 = concatenate([down1_pool, res_stream_pool], axis=3)\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower1 = down2\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower2 = down2\n",
    "    down2 = concatenate([tower1, tower2], axis=3)\n",
    "    down2 = Conv2D(24, (1, 1), padding='same')(down2)\n",
    "    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)\n",
    "\n",
    "    res2 = UpSampling2D((8,8))(down2)\n",
    "    res_stream = Add()([res2, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "    down3 = concatenate([down2_pool, res_stream_pool], axis=3)\n",
    "\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower1 = down3\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower2 = down3\n",
    "    down3 = concatenate([tower1, tower2], axis=3)\n",
    "    down3 = Conv2D(24, (1, 1), padding='same')(down3)\n",
    "    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)\n",
    "\n",
    "    res3 = UpSampling2D((16,16))(down3)\n",
    "    res_stream = Add()([res3, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    down4 = concatenate([down3_pool, res_stream_pool], axis=3)\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower1 = down4\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower2 = down4\n",
    "    down4 = concatenate([tower1, tower2], axis=3)\n",
    "    down4 = Conv2D(24, (1, 1), padding='same')(down4)\n",
    "    down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)\n",
    "\n",
    "    res4 = UpSampling2D((32,32))(down4)\n",
    "    res_stream = Add()([res4, res_stream])  #640\n",
    "\n",
    "\n",
    "    # 10x15\n",
    "    res_stream_pool = MaxPooling2D((64, 64), strides=(64, 64))(res_stream)\n",
    "\n",
    "    center = concatenate([down4_pool, res_stream_pool], axis=3)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(24, (1, 1), padding='same')(center)\n",
    "\n",
    "    resCen = UpSampling2D((64,64))(center)\n",
    "    res_stream = Add()([resCen, res_stream])  #640\n",
    "\n",
    "    # 20x30\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    up4 = UpSampling2D((2, 2))(center)\n",
    "    up4 = concatenate([res_stream_pool, up4], axis=3)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower1 = up4\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower2 = up4\n",
    "    up4 = concatenate([tower1,tower2], axis=3)\n",
    "    up4 = Conv2D(24, (1, 1), padding='same')(up4)\n",
    "\n",
    "    res_up4 = UpSampling2D((32, 32))(up4)\n",
    "    res_stream = Add()([res_up4, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 40x60\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "\n",
    "    up3 = UpSampling2D((2, 2))(up4)\n",
    "    up3 = concatenate([res_stream_pool, up3], axis=3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower1 = up3\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower2 = up3\n",
    "    up3 = concatenate([tower1, tower2], axis=3)\n",
    "    up3 = Conv2D(24, (1, 1), padding='same')(up3)\n",
    "\n",
    "    res_up3 = UpSampling2D((16, 16))(up3)\n",
    "    res_stream = Add()([res_up3, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 80x120\n",
    "    res_stream_pool = MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    up2 = UpSampling2D((2, 2))(up3)\n",
    "    up2 = concatenate([res_stream_pool, up2], axis=3)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower1 = up2\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower2 = up2\n",
    "    up2 = concatenate([tower1,tower2], axis=3)\n",
    "    up2 = Conv2D(24, (1, 1), padding='same')(up2)\n",
    "\n",
    "    res_up2 = UpSampling2D((8, 8))(up2)\n",
    "    res_stream = Add()([res_up2, res_stream])  # 640\n",
    "\n",
    "    # 160x240\n",
    "    res_stream_pool = MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    up1 = UpSampling2D((2, 2))(up2)\n",
    "    up1 = concatenate([res_stream_pool, up1], axis=3)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower1 = up1\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower2 = up1\n",
    "    up1 = concatenate([tower1,tower2],axis=3)\n",
    "    up1 = Conv2D(24, (1, 1), padding='same')(up1)\n",
    "\n",
    "    res_up1 = UpSampling2D((4, 4))(up1)\n",
    "    res_stream = Add()([res_up1, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 320x480\n",
    "    # 320x480\n",
    "    res_stream_pool = MaxPooling2D((2, 2), strides=(2, 2))(res_stream)\n",
    "\n",
    "    up0 = UpSampling2D((2, 2))(up1)\n",
    "    up0 = concatenate([res_stream_pool, up0, down0], axis=3)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower1 = up0\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower2 = up0\n",
    "    up0 = concatenate([tower1, tower2], axis=3)\n",
    "    up0 = Conv2D(24, (1, 1), padding='same')(up0)\n",
    "\n",
    "    res_up0 = UpSampling2D((2, 2))(up0)\n",
    "    res_stream = Add()([res_up0, res_stream])  # 640\n",
    "\n",
    "    # 640x960\n",
    "\n",
    "    up0a = UpSampling2D((2, 2))(up0)\n",
    "    up0a = concatenate([res_stream, down0a, up0a], axis=3)\n",
    "\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower1 = up0a\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower2 = up0a\n",
    "    up0a = concatenate([tower1, tower2], axis=3)\n",
    "    up0a = Conv2D(24, (1, 1), padding='same')(up0a)\n",
    "    # up0a = SpatialDropout2D(0.5)(up0a)\n",
    "\n",
    "    res_stream = Add()([up0a, res_stream])  # 640\n",
    "\n",
    "    feature_1 = Conv2D(16, (31, 1), activation='relu', padding='same', dilation_rate=(1, 1))(res_stream)\n",
    "    feature_1 = BatchNormalization()(feature_1)\n",
    "    feature_1 = Conv2D(16, (1, 31), activation='relu', padding='same', dilation_rate=(1, 1))(feature_1)\n",
    "    feature_1 = BatchNormalization()(feature_1)\n",
    "\n",
    "    feature_2 = Conv2D(16, (1, 31), activation='relu', padding='same', dilation_rate=(1, 1))(res_stream)\n",
    "    feature_2 = BatchNormalization()(feature_2)\n",
    "    feature_2 = Conv2D(16, (31, 1), activation='relu', padding='same', dilation_rate=(1, 1))(feature_2)\n",
    "    feature_2 = BatchNormalization()(feature_2)\n",
    "\n",
    "    feature = Add()([feature_1, feature_2])\n",
    "\n",
    "    main_feature = Conv2D(16, (3, 3), padding='same', use_bias=True, name = 'feature_main_cnv1')(feature)\n",
    "    main_feature = BatchNormalization()(main_feature)\n",
    "    main_feature = Activation('relu')(main_feature)\n",
    "    main_feature = Conv2D(16, (3, 3), padding='same', use_bias=True, name='feature_main_cnv2')(main_feature)\n",
    "\n",
    "    classify_main = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify_main')(main_feature)\n",
    "\n",
    "\n",
    "    #model = Model(inputs=inputs, outputs=[output,classify])\n",
    "    model = Model(inputs=inputs, outputs=classify_main)\n",
    "    #model = Model(inputs=inputs, outputs=[classify,tower1,tower2,tower3])\n",
    "\n",
    "    for i, layer in enumerate(model.layers):\n",
    "        print(i, layer.name)\n",
    "\n",
    "    model.summary()\n",
    "\n",
    "    model.compile(optimizer=Adam(lr=0.0005),\n",
    "                  loss=bce_dice_loss,\n",
    "                  #loss={'main_output': bce_dice_loss, 'aux_output': 'binary_crossentropy'},\n",
    "                  metrics=[dice_coeff])\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5},\n",
    "                  #loss_weights={'classify_main': 1., 'classify_edge': 0.2,  'classify_res': 0.2})\n",
    "\n",
    "    return model\n",
    "\n",
    "#########################################################################################3\n",
    "\n",
    "#############################################################################################\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_unet_256_256_DSB(input_shape=(256, 256, 3),\n",
    "                       num_classes=1):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Res block\n",
    "\n",
    "    input_conv = Conv2D(24, (3, 3), padding='same')(inputs)\n",
    "    input_conv = BatchNormalization()(input_conv)\n",
    "    input_conv= Activation('relu')(input_conv)\n",
    "\n",
    "    # 256\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(inputs)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower1 = down0a\n",
    "    down0a = Conv2D(16, (3, 3), padding='same')(down0a)\n",
    "    down0a = BatchNormalization()(down0a)\n",
    "    down0a = Activation('relu')(down0a)\n",
    "    tower2 = down0a\n",
    "    down0a = concatenate([tower1, tower2], axis=3)\n",
    "    down0a = Conv2D(24, (1, 1), padding='same')(down0a)\n",
    "    down0a_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0a)\n",
    "\n",
    "    res_stream = Add()([down0a, input_conv]) #640\n",
    "\n",
    "\n",
    "    # 128\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0a_pool)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower1 = down0\n",
    "    down0 = Conv2D(32, (3, 3), padding='same')(down0)\n",
    "    down0 = BatchNormalization()(down0)\n",
    "    down0 = Activation('relu')(down0)\n",
    "    tower2 = down0\n",
    "    down0 = concatenate([tower1, tower2], axis=3)\n",
    "    down0 = Conv2D(24, (1, 1), padding='same')(down0)\n",
    "    down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)\n",
    "\n",
    "    res0 = UpSampling2D((2,2))(down0)\n",
    "    res_stream = Add()([res0, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 64\n",
    "    res_stream_pool =  MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    down1 = concatenate([down0_pool, res_stream_pool], axis=3)\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower1 = down1\n",
    "    down1 = Conv2D(64, (3, 3), padding='same')(down1)\n",
    "    down1 = BatchNormalization()(down1)\n",
    "    down1 = Activation('relu')(down1)\n",
    "    tower2 = down1\n",
    "    down1 = concatenate([tower1, tower2], axis=3)\n",
    "    down1 = Conv2D(24, (1, 1), padding='same')(down1)\n",
    "    down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)\n",
    "\n",
    "    res1 = UpSampling2D((4,4))(down1) #640\n",
    "    res_stream = Add()([res1, res_stream])\n",
    "\n",
    "\n",
    "\n",
    "    # 32\n",
    "    res_stream_pool =  MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    down2 = concatenate([down1_pool, res_stream_pool], axis=3)\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower1 = down2\n",
    "    down2 = Conv2D(128, (3, 3), padding='same')(down2)\n",
    "    down2 = BatchNormalization()(down2)\n",
    "    down2 = Activation('relu')(down2)\n",
    "    tower2 = down2\n",
    "    down2 = concatenate([tower1, tower2], axis=3)\n",
    "    down2 = Conv2D(24, (1, 1), padding='same')(down2)\n",
    "    down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)\n",
    "\n",
    "    res2 = UpSampling2D((8,8))(down2)\n",
    "    res_stream = Add()([res2, res_stream])  #640\n",
    "\n",
    "\n",
    "\n",
    "    # 16\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "    down3 = concatenate([down2_pool, res_stream_pool], axis=3)\n",
    "\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower1 = down3\n",
    "    down3 = Conv2D(256, (3, 3), padding='same')(down3)\n",
    "    down3 = BatchNormalization()(down3)\n",
    "    down3 = Activation('relu')(down3)\n",
    "    tower2 = down3\n",
    "    down3 = concatenate([tower1, tower2], axis=3)\n",
    "    down3 = Conv2D(24, (1, 1), padding='same')(down3)\n",
    "    down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)\n",
    "\n",
    "    res3 = UpSampling2D((16,16))(down3)\n",
    "    res_stream = Add()([res3, res_stream])  #640\n",
    "\n",
    "    # 8\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    down4 = concatenate([down3_pool, res_stream_pool], axis=3)\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower1 = down4\n",
    "    down4 = Conv2D(512, (3, 3), padding='same')(down4)\n",
    "    down4 = BatchNormalization()(down4)\n",
    "    down4 = Activation('relu')(down4)\n",
    "    tower2 = down4\n",
    "    down4 = concatenate([tower1, tower2], axis=3)\n",
    "    down4 = Conv2D(24, (1, 1), padding='same')(down4)\n",
    "    down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)\n",
    "\n",
    "    res4 = UpSampling2D((32,32))(down4)\n",
    "    res_stream = Add()([res4, res_stream])  #640\n",
    "\n",
    "\n",
    "    # 4\n",
    "    res_stream_pool = MaxPooling2D((64, 64), strides=(64, 64))(res_stream)\n",
    "\n",
    "    center = concatenate([down4_pool, res_stream_pool], axis=3)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(1024, (3, 3), padding='same')(center)\n",
    "    center = BatchNormalization()(center)\n",
    "    center = Activation('relu')(center)\n",
    "    center = Conv2D(24, (1, 1), padding='same')(center)\n",
    "\n",
    "    resCen = UpSampling2D((64,64))(center)\n",
    "    res_stream = Add()([resCen, res_stream])  #640\n",
    "\n",
    "    # 8\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((32, 32), strides=(32, 32))(res_stream)\n",
    "\n",
    "    up4 = UpSampling2D((2, 2))(center)\n",
    "    up4 = concatenate([res_stream_pool, up4], axis=3)\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower1 = up4\n",
    "    up4 = Conv2D(512, (3, 3), padding='same')(up4)\n",
    "    up4 = BatchNormalization()(up4)\n",
    "    up4 = Activation('relu')(up4)\n",
    "    tower2 = up4\n",
    "    up4 = concatenate([tower1,tower2], axis=3)\n",
    "    up4 = Conv2D(24, (1, 1), padding='same')(up4)\n",
    "\n",
    "    res_up4 = UpSampling2D((32, 32))(up4)\n",
    "    res_stream = Add()([res_up4, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 16\n",
    "\n",
    "    res_stream_pool = MaxPooling2D((16, 16), strides=(16, 16))(res_stream)\n",
    "\n",
    "    up3 = UpSampling2D((2, 2))(up4)\n",
    "    up3 = concatenate([res_stream_pool, up3], axis=3)\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower1 = up3\n",
    "    up3 = Conv2D(256, (3, 3), padding='same')(up3)\n",
    "    up3 = BatchNormalization()(up3)\n",
    "    up3 = Activation('relu')(up3)\n",
    "    tower2 = up3\n",
    "    up3 = concatenate([tower1, tower2], axis=3)\n",
    "    up3 = Conv2D(24, (1, 1), padding='same')(up3)\n",
    "\n",
    "    res_up3 = UpSampling2D((16, 16))(up3)\n",
    "    res_stream = Add()([res_up3, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 32\n",
    "    res_stream_pool = MaxPooling2D((8, 8), strides=(8, 8))(res_stream)\n",
    "\n",
    "    up2 = UpSampling2D((2, 2))(up3)\n",
    "    up2 = concatenate([res_stream_pool, up2], axis=3)\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower1 = up2\n",
    "    up2 = Conv2D(128, (3, 3), padding='same')(up2)\n",
    "    up2 = BatchNormalization()(up2)\n",
    "    up2 = Activation('relu')(up2)\n",
    "    tower2 = up2\n",
    "    up2 = concatenate([tower1,tower2], axis=3)\n",
    "    up2 = Conv2D(24, (1, 1), padding='same')(up2)\n",
    "\n",
    "    res_up2 = UpSampling2D((8, 8))(up2)\n",
    "    res_stream = Add()([res_up2, res_stream])  # 640\n",
    "\n",
    "    # 64\n",
    "    res_stream_pool = MaxPooling2D((4, 4), strides=(4, 4))(res_stream)\n",
    "\n",
    "    up1 = UpSampling2D((2, 2))(up2)\n",
    "    up1 = concatenate([res_stream_pool, up1], axis=3)\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower1 = up1\n",
    "    up1 = Conv2D(64, (3, 3), padding='same')(up1)\n",
    "    up1 = BatchNormalization()(up1)\n",
    "    up1 = Activation('relu')(up1)\n",
    "    tower2 = up1\n",
    "    up1 = concatenate([tower1,tower2],axis=3)\n",
    "    up1 = Conv2D(24, (1, 1), padding='same')(up1)\n",
    "\n",
    "    res_up1 = UpSampling2D((4, 4))(up1)\n",
    "    res_stream = Add()([res_up1, res_stream])  # 640\n",
    "\n",
    "\n",
    "    # 128\n",
    "    res_stream_pool = MaxPooling2D((2, 2), strides=(2, 2))(res_stream)\n",
    "\n",
    "    up0 = UpSampling2D((2, 2))(up1)\n",
    "    up0 = concatenate([res_stream_pool, up0, down0], axis=3)\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower1 = up0\n",
    "    up0 = Conv2D(32, (3, 3), padding='same')(up0)\n",
    "    up0 = BatchNormalization()(up0)\n",
    "    up0 = Activation('relu')(up0)\n",
    "    tower2 = up0\n",
    "    up0 = concatenate([tower1,tower2],axis=3)\n",
    "    up0 = Conv2D(24, (1, 1), padding='same')(up0)\n",
    "\n",
    "    res_up0 = UpSampling2D((2, 2))(up0)\n",
    "    res_stream = Add()([res_up0, res_stream])  # 640\n",
    "\n",
    "    # 256\n",
    "\n",
    "    up0a = UpSampling2D((2, 2))(up0)\n",
    "    up0a = concatenate([res_stream, down0a, up0a], axis=3)\n",
    "\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower1 = up0a\n",
    "    up0a = Conv2D(16, (3, 3), padding='same')(up0a)\n",
    "    up0a = BatchNormalization()(up0a)\n",
    "    up0a = Activation('relu')(up0a)\n",
    "    tower2 = up0a\n",
    "    up0a = concatenate([tower1,tower2],axis=3)\n",
    "    up0a = Conv2D(24, (1, 1), padding='same')(up0a)\n",
    "    #up0a = SpatialDropout2D(0.5)(up0a)\n",
    "\n",
    "    res_stream = Add()([up0a, res_stream])  # 640\n",
    "    #keras.layers.convolutional.Conv2D(filters, kernel_size, strides=(1, 1), padding='valid', data_format=None,\n",
    "    #                                  dilation_rate=(1, 1), activation=None, use_bias=True,\n",
    "    #                                  kernel_initializer='glorot_uniform', bias_initializer='zeros',\n",
    "    #                                  kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None,\n",
    "    #                                  kernel_constraint=None, bias_constraint=None)\n",
    "\n",
    "    features = Conv2D(24, (3, 3), padding='same', use_bias=False)(res_stream)\n",
    "    classify = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True)(features)\n",
    "\n",
    "    #8 net experiment\n",
    "    #features2 = Conv2D(8, (5, 5), padding='same', use_bias=True, name = 'featurs2')(res_stream)\n",
    "    #classify2 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify2')(features2)\n",
    "    #features3 = Conv2D(8, (7, 7), padding='same', use_bias=True, name = 'featurs3')(res_stream)\n",
    "    #classify3 = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='classify3')(features3)\n",
    "\n",
    "    #main_feature = Average()([classify1,classify2,classify3])\n",
    "    #main_output = Conv2D(1, (1, 1), activation='sigmoid', use_bias=True, name='main_output')(main_feature)\n",
    "\n",
    "    ##crf = CRF( units=128,sparse_target=True, activation='sigmoid',name = 'crf')(classify) # for word embedding. one dimentional\n",
    "\n",
    "    #output = CrfRnnLayer(image_dims=(640, 960),\n",
    "    #                     num_classes=1,\n",
    "    #                     theta_alpha=160.,\n",
    "    #                     theta_beta=3.,\n",
    "    #                     theta_gamma=3.,\n",
    "    #                     num_iterations=5,\n",
    "    #                     name='crfrnn')([classify, inputs])\n",
    "\n",
    "    #model = Model(inputs=inputs, outputs=[output,classify])\n",
    "    model = Model(inputs=inputs, outputs=classify)\n",
    "    #model = Model(inputs=inputs, outputs=[classify,tower1,tower2,tower3])\n",
    "\n",
    "    for i, layer in enumerate(model.layers):\n",
    "        print(i, layer.name)\n",
    "\n",
    "    model.summary()\n",
    "\n",
    "    # model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss=bce_dice_loss, metrics=[dice_loss])\n",
    "    #model.compile(optimizer=Adam(lr=0.0001), loss=bce_dice_loss, metrics=[dice_coeff])\n",
    "\n",
    "    model.compile(optimizer=Adam(lr=0.0005),\n",
    "                  loss=bce_dice_loss,\n",
    "                  #loss={'main_output': bce_dice_loss, 'aux_output': 'binary_crossentropy'},\n",
    "                  metrics=[mean_iou])\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5},\n",
    "                  #loss_weights={'main_output': 1., 'aux_output': 0.5})\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Unet(img_size):\n",
    "    inputs = Input((img_size, img_size, 3))\n",
    "    s = Lambda(lambda x: x / 255)(inputs)\n",
    "\n",
    "    c1 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(s)\n",
    "    c1 = Dropout(0.1)(c1)\n",
    "    c1 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c1)\n",
    "    p1 = MaxPooling2D((2, 2))(c1)\n",
    "\n",
    "    c2 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p1)\n",
    "    c2 = Dropout(0.1)(c2)\n",
    "    c2 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c2)\n",
    "    p2 = MaxPooling2D((2, 2))(c2)\n",
    "    c3 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p2)\n",
    "    c3 = Dropout(0.2)(c3)\n",
    "    c3 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c3)\n",
    "    p3 = MaxPooling2D((2, 2))(c3)\n",
    "\n",
    "    c4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p3)\n",
    "    c4 = Dropout(0.2)(c4)\n",
    "    c4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c4)\n",
    "    p4 = MaxPooling2D(pool_size=(2, 2))(c4)\n",
    "\n",
    "    c5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p4)\n",
    "    c5 = Dropout(0.3)(c5)\n",
    "    c5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c5)\n",
    "\n",
    "    u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)\n",
    "    u6 = concatenate([u6, c4])\n",
    "    c6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u6)\n",
    "    c6 = Dropout(0.2)(c6)\n",
    "    c6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c6)\n",
    "\n",
    "    u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)\n",
    "    u7 = concatenate([u7, c3])\n",
    "    c7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u7)\n",
    "    c7 = Dropout(0.2)(c7)\n",
    "    c7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c7)\n",
    "\n",
    "    u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)\n",
    "    u8 = concatenate([u8, c2])\n",
    "    c8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u8)\n",
    "    c8 = Dropout(0.1)(c8)\n",
    "    c8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c8)\n",
    "\n",
    "    u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)\n",
    "    u9 = concatenate([u9, c1], axis=3)\n",
    "    c9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u9)\n",
    "    c9 = Dropout(0.1)(c9)\n",
    "    c9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c9)\n",
    "\n",
    "    outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)\n",
    "\n",
    "    model = Model(inputs=[inputs], outputs=[outputs])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}