190213_임베딩과 표상학습.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 임베딩을 학습하는 것은 효과적인 압축 알고리즘을 개발하는 것과 같다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 데이텃셋에 대해 가장 많은 정보를 전달하는 축의 집합을 찾고 싶다는것.\n",
    "# 비선형적인 복잡한 데이터셋에서 중요한 관계를 포착하기 어려움(이미지, 텍스트)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 오토인코더 구조의 동기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 입력부분에서 중요한 요소들에 대해 상당한 양의 정보 손실이 있는데\n",
    "# 이는 입력을 받고 저차원 벡터로 압축한다. \n",
    "# 인코더는 저차원 임베딩 또는 코드 생성이고 \n",
    "# 디코더는 임의의 레이블에 임베딩을 맞추는 대신 신경망을 첫 절반 계산을 반전하고 \n",
    "# 원래 입력을 재구성하려고 시도하는부분이다.\n",
    "# 입력 -> 인코더 -> 임베딩 -> 디코더 -> 재구성 입력\n",
    "# 오토인코더 구조는 고차원 입력을 저차원 임베딩으로 구성하려고 시도하고 그다음 입력\n",
    "# 을 재구성하기 위해 저차원 임베딩을 사용한다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 텐서플로로 오토인코더 구현하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting input_data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  Could not find a version that satisfies the requirement input_data (from versions: )\n",
      "No matching distribution found for input_data\n",
      "You are using pip version 18.1, however version 19.0.2 is available.\n",
      "You should consider upgrading via the 'python -m pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "!pip install input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'input_data'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-ee1082e4f863>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0minput_data\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mcontrol_flow_ops\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margparse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'input_data'"
     ]
    }
   ],
   "source": [
    "import input_data\n",
    "import tensorflow as tf\n",
    "from tensorflow.python import control_flow_ops\n",
    "import time, argparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'input_data'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-27118c629225>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0minput_data\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mcontrol_flow_ops\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margparse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'input_data'"
     ]
    }
   ],
   "source": [
    "# Architecture\n",
    "n_encoder_hidden_1 = 1000\n",
    "n_encoder_hidden_2 = 500\n",
    "n_encoder_hidden_3 = 250\n",
    "n_decoder_hidden_1 = 250\n",
    "n_decoder_hidden_2 = 500\n",
    "n_decoder_hidden_3 = 1000\n",
    "\n",
    "# Parameters\n",
    "learning_rate = 0.01\n",
    "training_epochs = 1000\n",
    "batch_size = 100\n",
    "display_step = 1\n",
    "\n",
    "def layer_batch_norm(x, n_out, phase_train):\n",
    "    beta_init = tf.constant_initializer(value=0.0, dtype=tf.float32)\n",
    "    gamma_init = tf.constant_initializer(value=1.0, dtype=tf.float32)\n",
    "\n",
    "    beta = tf.get_variable(\"beta\", [n_out], initializer=beta_init)\n",
    "    gamma = tf.get_variable(\"gamma\", [n_out], initializer=gamma_init)\n",
    "\n",
    "    batch_mean, batch_var = tf.nn.moments(x, [0], name='moments')\n",
    "    ema = tf.train.ExponentialMovingAverage(decay=0.9)\n",
    "    ema_apply_op = ema.apply([batch_mean, batch_var])\n",
    "    ema_mean, ema_var = ema.average(batch_mean), ema.average(batch_var)\n",
    "    def mean_var_with_update():\n",
    "        with tf.control_dependencies([ema_apply_op]):\n",
    "            return tf.identity(batch_mean), tf.identity(batch_var)\n",
    "    mean, var = control_flow_ops.cond(phase_train,\n",
    "        mean_var_with_update,\n",
    "        lambda: (ema_mean, ema_var))\n",
    "\n",
    "    reshaped_x = tf.reshape(x, [-1, 1, 1, n_out])\n",
    "    normed = tf.nn.batch_norm_with_global_normalization(reshaped_x, mean, var,\n",
    "        beta, gamma, 1e-3, True)\n",
    "    return tf.reshape(normed, [-1, n_out])\n",
    "\n",
    "def layer(input, weight_shape, bias_shape, phase_train):\n",
    "    weight_init = tf.random_normal_initializer(stddev=(1.0/weight_shape[0])**0.5)\n",
    "    bias_init = tf.constant_initializer(value=0)\n",
    "    W = tf.get_variable(\"W\", weight_shape,\n",
    "                        initializer=weight_init)\n",
    "    b = tf.get_variable(\"b\", bias_shape,\n",
    "                        initializer=bias_init)\n",
    "    logits = tf.matmul(input, W) + b\n",
    "    return tf.nn.sigmoid(layer_batch_norm(logits, weight_shape[1], phase_train))\n",
    "\n",
    "def encoder(x, n_code, phase_train):\n",
    "    with tf.variable_scope(\"encoder\"):\n",
    "        with tf.variable_scope(\"hidden_1\"):\n",
    "            hidden_1 = layer(x, [784, n_encoder_hidden_1], [n_encoder_hidden_1], phase_train)\n",
    "\n",
    "        with tf.variable_scope(\"hidden_2\"):\n",
    "            hidden_2 = layer(hidden_1, [n_encoder_hidden_1, n_encoder_hidden_2], [n_encoder_hidden_2], phase_train)\n",
    "\n",
    "        with tf.variable_scope(\"hidden_3\"):\n",
    "            hidden_3 = layer(hidden_2, [n_encoder_hidden_2, n_encoder_hidden_3], [n_encoder_hidden_3], phase_train)\n",
    "\n",
    "        with tf.variable_scope(\"code\"):\n",
    "            code = layer(hidden_3, [n_encoder_hidden_3, n_code], [n_code], phase_train)\n",
    "\n",
    "    return code\n",
    "\n",
    "def decoder(code, n_code, phase_train):\n",
    "    with tf.variable_scope(\"decoder\"):\n",
    "        with tf.variable_scope(\"hidden_1\"):\n",
    "            hidden_1 = layer(code, [n_code, n_decoder_hidden_1], [n_decoder_hidden_1], phase_train)\n",
    "\n",
    "        with tf.variable_scope(\"hidden_2\"):\n",
    "            hidden_2 = layer(hidden_1, [n_decoder_hidden_1, n_decoder_hidden_2], [n_decoder_hidden_2], phase_train)\n",
    "\n",
    "        with tf.variable_scope(\"hidden_3\"):\n",
    "            hidden_3 = layer(hidden_2, [n_decoder_hidden_2, n_decoder_hidden_3], [n_decoder_hidden_3], phase_train)\n",
    "\n",
    "        with tf.variable_scope(\"output\"):\n",
    "            output = layer(hidden_3, [n_decoder_hidden_3, 784], [784], phase_train)\n",
    "\n",
    "    return output\n",
    "\n",
    "def loss(output, x):\n",
    "    with tf.variable_scope(\"training\"):\n",
    "        l2 = tf.sqrt(tf.reduce_sum(tf.square(tf.sub(output, x)), 1))\n",
    "        train_loss = tf.reduce_mean(l2)\n",
    "        train_summary_op = tf.scalar_summary(\"train_cost\", train_loss)\n",
    "        return train_loss, train_summary_op\n",
    "\n",
    "def training(cost, global_step):\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-08,\n",
    "        use_locking=False, name='Adam')\n",
    "    train_op = optimizer.minimize(cost, global_step=global_step)\n",
    "    return train_op\n",
    "\n",
    "def image_summary(label, tensor):\n",
    "    tensor_reshaped = tf.reshape(tensor, [-1, 28, 28, 1])\n",
    "    return tf.image_summary(label, tensor_reshaped)\n",
    "\n",
    "def evaluate(output, x):\n",
    "    with tf.variable_scope(\"validation\"):\n",
    "        in_im_op = image_summary(\"input_image\", x)\n",
    "        out_im_op = image_summary(\"output_image\", output)\n",
    "        l2 = tf.sqrt(tf.reduce_sum(tf.square(tf.sub(output, x, name=\"val_diff\")), 1))\n",
    "        val_loss = tf.reduce_mean(l2)\n",
    "        val_summary_op = tf.scalar_summary(\"val_cost\", val_loss)\n",
    "        return val_loss, in_im_op, out_im_op, val_summary_op\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "\n",
    "    parser = argparse.ArgumentParser(description='Test various optimization strategies')\n",
    "    parser.add_argument('n_code', nargs=1, type=str)\n",
    "    args = parser.parse_args()\n",
    "    n_code = args.n_code[0]\n",
    "\n",
    "    mnist = input_data.read_data_sets(\"data/\", one_hot=True)\n",
    "\n",
    "    with tf.Graph().as_default():\n",
    "\n",
    "        with tf.variable_scope(\"autoencoder_model\"):\n",
    "\n",
    "            x = tf.placeholder(\"float\", [None, 784]) # mnist data image of shape 28*28=784\n",
    "            phase_train = tf.placeholder(tf.bool)\n",
    "\n",
    "            code = encoder(x, int(n_code), phase_train)\n",
    "\n",
    "            output = decoder(code, int(n_code), phase_train)\n",
    "\n",
    "            cost, train_summary_op = loss(output, x)\n",
    "\n",
    "            global_step = tf.Variable(0, name='global_step', trainable=False)\n",
    "\n",
    "            train_op = training(cost, global_step)\n",
    "\n",
    "            eval_op, in_im_op, out_im_op, val_summary_op = evaluate(output, x)\n",
    "\n",
    "            summary_op = tf.merge_all_summaries()\n",
    "\n",
    "            saver = tf.train.Saver(max_to_keep=200)\n",
    "\n",
    "            sess = tf.Session()\n",
    "\n",
    "            train_writer = tf.train.SummaryWriter(\"mnist_autoencoder_hidden=\" + n_code + \"_logs/\",\n",
    "                                                graph=sess.graph)\n",
    "\n",
    "            val_writer = tf.train.SummaryWriter(\"mnist_autoencoder_hidden=\" + n_code + \"_logs/\",\n",
    "                                                graph=sess.graph)\n",
    "\n",
    "            init_op = tf.initialize_all_variables()\n",
    "\n",
    "            sess.run(init_op)\n",
    "\n",
    "            # Training cycle\n",
    "            for epoch in range(training_epochs):\n",
    "\n",
    "                avg_cost = 0.\n",
    "                total_batch = int(mnist.train.num_examples/batch_size)\n",
    "                # Loop over all batches\n",
    "                for i in range(total_batch):\n",
    "                    minibatch_x, minibatch_y = mnist.train.next_batch(batch_size)\n",
    "                    # Fit training using batch data\n",
    "                    _, new_cost, train_summary = sess.run([train_op, cost, train_summary_op], feed_dict={x: minibatch_x, phase_train: True})\n",
    "                    train_writer.add_summary(train_summary, sess.run(global_step))\n",
    "                    # Compute average loss\n",
    "                    avg_cost += new_cost/total_batch\n",
    "                # Display logs per epoch step\n",
    "                if epoch % display_step == 0:\n",
    "                    print (\"Epoch:\", '%04d' % (epoch+1), \"cost =\", \"{:.9f}\".format(avg_cost))\n",
    "\n",
    "                    train_writer.add_summary(train_summary, sess.run(global_step))\n",
    "\n",
    "                    validation_loss, in_im, out_im, val_summary = sess.run([eval_op, in_im_op, out_im_op, val_summary_op], feed_dict={x: mnist.validation.images, phase_train: False})\n",
    "                    val_writer.add_summary(in_im, sess.run(global_step))\n",
    "                    val_writer.add_summary(out_im, sess.run(global_step))\n",
    "                    val_writer.add_summary(val_summary, sess.run(global_step))\n",
    "                    print(\"Validation Loss:\", validation_loss)\n",
    "\n",
    "                    saver.save(sess, \"mnist_autoencoder_hidden=\" + n_code + \"_logs/model-checkpoint-\" + '%04d' % (epoch+1), global_step=global_step)\n",
    "\n",
    "\n",
    "            print(\"Optimization Finished!\")\n",
    "\n",
    "\n",
    "            test_loss = sess.run(eval_op, feed_dict={x: mnist.test.images, phase_train: False})\n",
    "\n",
    "            print(\"Test Loss:\", loss)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}