t3d_train_eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# def compose(*funcs):\n",
    "#     if funcs:\n",
    "#         return reduce(lambda f,g:lambda *a,**kw:g(f(*a,**kw)),funcs)\n",
    "#     else:\n",
    "#         raise ValueError('No function to compose.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# class A(object):\n",
    "#     def __init__(self):\n",
    "#         print(\"A\")\n",
    "# class B(object):\n",
    "#     def __init__(self):\n",
    "#         print(\"B\")\n",
    "# class C(object):\n",
    "#     def __init__(self):\n",
    "#         super(C,self).__init__()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import tensorflow as tf\n",
    "import random\n",
    "import numpy as np\n",
    "import time\n",
    "import os\n",
    "import tensorflow.contrib.slim as slim\n",
    "import DataGenerator\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE=10\n",
    "BN_SIZE=4\n",
    "GROWTH_RATE=32\n",
    "KEEP_PROB=1.0\n",
    "START_CHANNEL=64\n",
    "IS_TRAIN=True\n",
    "CROP_SIZE=160\n",
    "NUM_FRAMES_PER_CLIP=16\n",
    "NUM_CLASSES=8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def convS(_X,out_channels,kernel_size=[1,3,3],stride=1,padding='VALID'):\n",
    "    \n",
    "#     weights_S= slim.model_variable('weights', shape=[1,kernel_size[0],kernel_size[1],kernel_size[2],1], \n",
    "#                         initializer=tf.contrib.layers.xavier_initializer(), \n",
    "#                         regularizer=slim.l2_regularizer(0.0005),\n",
    "#                         device='/CPU:0')\n",
    "#     return tf.nn.conv3d(l_input, weights_S, strides=[1, stride, stride, stride, 1], padding=padding)\n",
    "    \n",
    "    return slim.conv3d(_X,out_channels,kernel_size=kernel_size,stride=stride,padding=padding,\n",
    "                                     weights_initializer=tf.truncated_normal_initializer(stddev=0.01),\n",
    "                                   biases_initializer=None)\n",
    "\n",
    "def convT(_X,out_channels,kernel_size=[3,1,1],stride=1,padding='VALID'):\n",
    "    return slim.conv3d(_X,out_channels,kernel_size=kernel_size,stride=stride,padding=padding,\n",
    "                         weights_initializer=tf.truncated_normal_initializer(stddev=0.01),\n",
    "                       biases_initializer=None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_accuracy(logit,labels):\n",
    "    correct=tf.equal(tf.argmax(logit,1),labels)\n",
    "    acc=tf.reduce_mean(tf.cast(correct,tf.float32))\n",
    "    return acc\n",
    "def compute_loss(logit,label):\n",
    "    cross_entropy_mean=tf.reduce_mean(\n",
    "                    tf.nn.sparse_softmax_cross_entropy_with_logits(labels=label,logits=logit))\n",
    "    weight_loss=tf.losses.get_regularization_loss()\n",
    "    total_loss=cross_entropy_mean+weight_loss\n",
    "    return total_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# def ST_A(_X,out_channels,s_kernel,t_kernel,stride,padding):\n",
    "#     x=convS(_X,out_channels,s_kernel,stride,padding)\n",
    "#     x=tf.layers.batch_normalization(x,training=IS_TRAIN)\n",
    "#     x=tf.nn.relu(x)\n",
    "#     x=convT(x,out_channels,t_kernel,stride,padding)\n",
    "#     x=tf.layers.batch_normalization(x,training=IS_TRAIN)\n",
    "#     x=tf.nn.relu(x)\n",
    "#     return x\n",
    "    \n",
    "def ST_B(_X,out_channels,s_kernel,t_kernel,stride,padding):\n",
    "    \n",
    "   \n",
    "    tmp_x=convS(_X,out_channels,s_kernel,stride,padding)\n",
    "    tmp_x=tf.layers.batch_normalization(tmp_x,training=IS_TRAIN)\n",
    "    tmp_x=tf.nn.relu(tmp_x)\n",
    "    \n",
    "    x=convT(_X,out_channels,t_kernel,stride,padding)\n",
    "    x=tf.layers.batch_normalization(x,training=IS_TRAIN)\n",
    "    x=tf.nn.relu(x)\n",
    "    \n",
    "    return x+tmp_x\n",
    "    \n",
    "def ST_C(_X,out_channels,s_kernel,t_kernel,stride,padding):\n",
    "    \n",
    "    x=convS(_X,out_channels,s_kernel,stride,padding)\n",
    "    x=tf.layers.batch_normalization(x,training=IS_TRAIN)\n",
    "    x=tf.nn.relu(x)\n",
    "    \n",
    "   \n",
    "    tmp_x=convT(x,out_channels,t_kernel,stride,padding)\n",
    "    tmp_x=tf.layers.batch_normalization(tmp_x,training=IS_TRAIN)\n",
    "    tmp_x=tf.nn.relu(tmp_x)\n",
    "    \n",
    "    return x+tmp_x\n",
    "def p3d(_X,out_channels,kernel_size=[3,3,3],stride=1,padding='SAME',tp='B'):\n",
    "    s_kernel=kernel_size[1:3]\n",
    "    s_kernel.insert(0,1)\n",
    "    t_kernel=[1,1]\n",
    "    t_kernel.insert(0,kernel_size[0])\n",
    "    if tp=='B':\n",
    "        return ST_B(_X,out_channels,s_kernel,t_kernel,stride,padding)\n",
    "    else:\n",
    "        return ST_C(_X,out_channels,s_kernel,t_kernel,stride,padding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_block_layer(_X,nums_inchannel,keep_prob=1,tp_idx=0):\n",
    "    #Caution: Do Not use tf.contrib.layers.batch_norm() !!!\n",
    "    out=tf.layers.batch_normalization(_X,training=IS_TRAIN)\n",
    "    out=tf.nn.relu(out)\n",
    "    out=slim.conv3d(out,BN_SIZE*GROWTH_RATE,[1,1,1],stride=1,biases_initializer=None)\n",
    "    #Here,we use 'pseudo-3d convolution' instead of traditional 3D convolution(eg. slim.conv3d)\n",
    "    if tp_idx % 2==0:\n",
    "        out=p3d(out,GROWTH_RATE,padding='SAME',tp='B')\n",
    "    else:\n",
    "        out=p3d(out,GROWTH_RATE,padding='SAME',tp='C')\n",
    "        #out=slim.conv3d(out,GROWTH_RATE,[3,3,3],stride=1,padding='SAME',biases_initializer=None)\n",
    "    if(keep_prob!=1):\n",
    "        out=slim.dropout(out,keep_prob=keep_prob)\n",
    "    return tf.concat([_X,out],axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_block(_X,block_num,nums_inchannel):\n",
    "    tp_idx=0\n",
    "    for i in range(block_num):\n",
    "        _X=make_block_layer(_X,nums_inchannel=nums_inchannel,keep_prob=KEEP_PROB,tp_idx=tp_idx)\n",
    "        tp_idx+=1\n",
    "    return _X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def TTL(_X,depth=(1,3,4)):\n",
    "    #As mentioned above,I replace traditional 3D conv by 'pseudo-3d conv' to achive parameters efficiency.\n",
    "    y1=tf.layers.batch_normalization(_X,training=IS_TRAIN)\n",
    "    y1=tf.nn.relu(y1)\n",
    "    y1=slim.conv3d(y1,128,[depth[0],1,1],stride=1,biases_initializer=None) \n",
    "    \n",
    "    \n",
    "#     y2=tf.layers.batch_normalization(_X,training=IS_TRAIN)\n",
    "#     y2=tf.nn.relu(y2)\n",
    "    y2=p3d(_X,128,[depth[1],3,3],stride=1,tp='B')\n",
    "    #y2=slim.conv3d(y2,128,[depth[1],3,3],stride=1,biases_initializer=None)\n",
    "    \n",
    "#     y3=tf.layers.batch_normalization(_X,training=IS_TRAIN)\n",
    "#     y3=tf.nn.relu(y3)\n",
    "    y3=p3d(_X,128,[depth[2],3,3],stride=1,tp='C')\n",
    "    #y3=slim.conv3d(y3,128,[depth[2],3,3],stride=1,biases_initializer=None)\n",
    "    \n",
    "#     y4=tf.layers.batch_normalization(_X,training=IS_TRAIN)\n",
    "#     y4=tf.nn.relu(y4)\n",
    "    \n",
    "    return tf.concat([y1,y2,y3],axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Transition(_X,in_channels):\n",
    "    _X=tf.layers.batch_normalization(_X,training=IS_TRAIN)\n",
    "    _X=tf.nn.relu(_X)\n",
    "    _X=slim.conv3d(_X,in_channels,kernel_size=[1,1,1],stride=1,biases_initializer=None)\n",
    "    _X=slim.avg_pool3d(_X,[2,2,2],stride=2,padding='SAME')\n",
    "    return _X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def count2():\n",
    "    print np.sum([np.prod(v.get_shape().as_list()) for v in tf.trainable_variables()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def inference_t3d(_X,block_config=(6, 12, 24, 16)):\n",
    "    \n",
    "    with slim.arg_scope([slim.conv3d],\n",
    "                       weights_initializer=tf.truncated_normal_initializer(stddev=0.01),\n",
    "                        weights_regularizer=slim.l2_regularizer(0.0005)): \n",
    "        out=slim.conv3d(_X,START_CHANNEL,[3,7,7],stride=[1,2,2],padding='SAME',biases_initializer=None)\n",
    "       \n",
    "        out=tf.layers.batch_normalization(out,training=IS_TRAIN)\n",
    "        out=tf.nn.relu(out)\n",
    "        out=slim.max_pool3d(out,kernel_size=[3,3,3],stride=2,padding='SAME')\n",
    "        in_channels=START_CHANNEL\n",
    "        \n",
    "        for i, num_layers in enumerate(block_config):\n",
    "            \n",
    "            out=build_block(out,num_layers,in_channels)\n",
    "            \n",
    "            in_channels=in_channels+GROWTH_RATE*num_layers\n",
    "            if i!=len(block_config)-1:\n",
    "                if i==0:\n",
    "                    out=TTL(out,(1,3,6))\n",
    "                else:\n",
    "                    out=TTL(out)\n",
    "                \n",
    "                in_channels=128*3\n",
    "                out=Transition(out,in_channels // 2)\n",
    "                in_channels=in_channels // 2\n",
    "                \n",
    "                \n",
    "        \n",
    "        out=tf.layers.batch_normalization(out,training=IS_TRAIN)\n",
    "        out=tf.nn.relu(out)\n",
    "        #Standard input shape=[BATCH,NUM_CLIP=16,HEIGHT=160,WIDTH=160,RGB=3],makes that kernel_size of AVG_POOL equals '5'\n",
    "        #If you are about to change size of input,changing the kernel size of 'avg_pool3d' simultaneously.\n",
    "        out=slim.avg_pool3d(out,kernel_size=[1,5,5])\n",
    "        out=tf.reshape(out,[out.get_shape().as_list()[0],-1])\n",
    "        \n",
    "        out=slim.fully_connected(out,NUM_CLASSES)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#define dataloader to fetch data for each batch \n",
    "dataloader=DataGenerator.DataGenerator(filename='train.list',\n",
    "                                batch_size=BATCH_SIZE,\n",
    "                                num_frames_per_clip=NUM_FRAMES_PER_CLIP,\n",
    "                                shuffle=True,is_da=False)\n",
    "# #function to generate one-hot label\n",
    "# def convertonehot(label):\n",
    "#     return (np.arange(NUM_CLASSES)==label[:,None]).astype(np.integer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(BATCH_SIZE)\n",
    "print(IS_TRAIN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cell for Training!\n",
    "USE_PRETRAIN=False\n",
    "MODEL_PATH='./'\n",
    "MAX_STEPS=5000\n",
    "\n",
    "with tf.Graph().as_default():\n",
    "    global_step=tf.get_variable('global_step',[],initializer=tf.constant_initializer(0),trainable=False)\n",
    "    input_placeholder=tf.placeholder(tf.float32,shape=[BATCH_SIZE,NUM_FRAMES_PER_CLIP,CROP_SIZE,CROP_SIZE,3])\n",
    "    label_placeholder=tf.placeholder(tf.int64,shape=[BATCH_SIZE])\n",
    "\n",
    "    logit=inference_t3d(input_placeholder)\n",
    "    \n",
    "    #define loss:\n",
    "    loss=compute_loss(logit, label_placeholder)\n",
    "    tf.summary.scalar('loss',loss)\n",
    "#     tf.summary.scalar('loss_total',total_loss)\n",
    "    acc=compute_accuracy(logit,label_placeholder)\n",
    "    tf.summary.scalar('accuracy',acc)\n",
    "    \n",
    "    #define lr dacay and optimizer:\n",
    "    learning_rate = tf.train.exponential_decay(0.0002,\n",
    "                                               global_step,decay_steps=450,decay_rate=0.3,staircase=True)\n",
    "    opt_stable=tf.train.AdamOptimizer(learning_rate)\n",
    "    \n",
    "    #build dependecy which updating paras before training:\n",
    "    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "    with tf.control_dependencies(update_ops):\n",
    "        optim_op=opt_stable.minimize(loss,global_step=global_step,var_list=tf.trainable_variables())\n",
    "    \n",
    "    saver=tf.train.Saver(tf.global_variables())\n",
    "    init=tf.global_variables_initializer()\n",
    "    config=tf.ConfigProto()\n",
    "    gpu_options = tf.GPUOptions(allow_growth=True)\n",
    "    sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))  \n",
    "    sess.run(init)\n",
    "   \n",
    "    if USE_PRETRAIN:\n",
    "        saver.restore(sess,MODEL_PATH)\n",
    "        print('Checkpoint reloaded.')\n",
    "    else:\n",
    "        print('Train from scratch.')\n",
    "    merged=tf.summary.merge_all()\n",
    "    #Using tensorboard to trace your training path.\n",
    "    train_writer=tf.summary.FileWriter('./visual_t3d/train',sess.graph)\n",
    "#     test_writer=tf.summary.FileWriter('./visual_logs/test',sess.graph)\n",
    "    \n",
    "    duration=0\n",
    "    print('Start training.')\n",
    "    for step in xrange(1,MAX_STEPS):\n",
    "        \n",
    "        start_time=time.time()\n",
    "        train_images,train_labels,_=dataloader.next_batch()\n",
    "        sess.run(optim_op,feed_dict={\n",
    "                        input_placeholder:train_images,\n",
    "                        label_placeholder:train_labels})\n",
    "        duration+=time.time()-start_time\n",
    "        \n",
    "        if step!=0 and step % 10==0:\n",
    "            curacc,curloss=sess.run([acc,loss],feed_dict={\n",
    "                        input_placeholder:train_images,\n",
    "                        label_placeholder:train_labels})\n",
    "            print('Step %d: %.2f sec -->loss : %.4f =====acc : %.2f' % (step, duration,np.mean(curloss),curacc))\n",
    "            duration=0\n",
    "            \n",
    "        if step!=0 and step % 50==0:\n",
    "            mer=sess.run(merged,feed_dict={\n",
    "                        input_placeholder:train_images,\n",
    "                        label_placeholder:train_labels})\n",
    "            train_writer.add_summary(mer, step)\n",
    "            \n",
    "        if step >1000 and step % 500==0 or (step+1)==MAX_STEPS:\n",
    "            saver.save(sess,'./T3DBN_TFCKPT_ITER_{}'.format(step),global_step=step)\n",
    "        \n",
    "    print('done')   \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "###############################################\n",
    "###############################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cell for Testing!\n",
    "BATCH_SIZE=10\n",
    "#Essential:set 'IS_TRAIN' -> False\n",
    "IS_TRAIN=False\n",
    "testloader=DataGenerator.DataGenerator(filename='test.list',\n",
    "                                batch_size=BATCH_SIZE,\n",
    "                                num_frames_per_clip=NUM_FRAMES_PER_CLIP,\n",
    "                                shuffle=False,is_da=False)\n",
    "tf.reset_default_graph()\n",
    "print('IS_TRAIN:',IS_TRAIN)\n",
    "print('KEEP_PROB:',KEEP_PROB)\n",
    "USE_PRETRAIN=True\n",
    "# MODEL_PATH='./GT_TFCKPT_ITER_3000-3000'\n",
    "with tf.Graph().as_default():\n",
    "    global_step=tf.get_variable('global_step',[],initializer=tf.constant_initializer(0),trainable=False)\n",
    "    input_placeholder=tf.placeholder(tf.float32,shape=[BATCH_SIZE,NUM_FRAMES_PER_CLIP,CROP_SIZE,CROP_SIZE,3])\n",
    "    label_placeholder=tf.placeholder(tf.int64,shape=[BATCH_SIZE])\n",
    "\n",
    "    logit=inference_t3d(input_placeholder)\n",
    "    \n",
    "    acc=compute_accuracy(logit,label_placeholder)\n",
    "    init=tf.global_variables_initializer()\n",
    "    saver=tf.train.Saver(tf.global_variables())\n",
    "    \n",
    "    config=tf.ConfigProto()\n",
    "    gpu_options = tf.GPUOptions(allow_growth=True)\n",
    "    sess = tf.Session()  \n",
    "    sess.run(init)\n",
    "   \n",
    "    saver.restore(sess,'./T3DBN_TFCKPT_ITER_3000-3000')\n",
    "    print('Start Testing.')\n",
    "    \n",
    "    print('l={}'.format(testloader.len/BATCH_SIZE))\n",
    "    resacc=0\n",
    "    c=0\n",
    "    for step in xrange(testloader.len/BATCH_SIZE):\n",
    "        \n",
    "        \n",
    "        test_images,test_labels,_=testloader.next_batch()\n",
    "        \n",
    "        curacc=sess.run(acc,feed_dict={\n",
    "                        input_placeholder:test_images,\n",
    "                        label_placeholder:test_labels})\n",
    "        print(curacc)\n",
    "        resacc+=curacc\n",
    "        c+=1\n",
    "        \n",
    "    print('Accuracy:{:.2f}'.format(resacc/c))   \n",
    "            "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}