07- Convolutional Neural Networks/CNN_edge_detection_sample.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The code is adapted from: https://d2l.ai/chapter_convolutional-neural-networks/conv-layer.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 1., 0., 0., 0., 0., 1., 1.],\n",
       "        [1., 1., 0., 0., 0., 0., 1., 1.],\n",
       "        [1., 1., 0., 0., 0., 0., 1., 1.],\n",
       "        [1., 1., 0., 0., 0., 0., 1., 1.],\n",
       "        [1., 1., 0., 0., 0., 0., 1., 1.],\n",
       "        [1., 1., 0., 0., 0., 0., 1., 1.]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = torch.ones((6, 8))\n",
    "X[:, 2:6] = 0\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAT0AAAD4CAYAAACeyTEuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAK2klEQVR4nO3d34vldR3H8dfL2RVzS7zYg4grTYEIIuTqYSMWpNaKNUW73AW9iGC70FAKRLsJ/wHxJoJFLcNfVCqImCmomJA/zq6arqthsuG01h6J0O0i0V5dzFkYbdw50vcz32++nw8Ydmbny5kX++O53/M9Z/Y4iQCgihP6HgAA64noASiF6AEohegBKIXoAShlQ4sb3bx5cxYXF1vc9KfC8/v29T0B/+e2XnBB3xMG7dChQ3r77be92ueaRG9xcVGTyaTFTX8qbPKqvxfA3Pj7dXzj8fhjP8fdWwClED0ApRA9AKUQPQClED0ApRA9AKUQPQClED0ApRA9AKUQPQClED0ApRA9AKUQPQClzBU92zttv2b7ddvXtx4FAK2sGT3bC5J+IuliSedI2m37nNbDAKCFec70tkl6PckbSd6TdI+ky9vOAoA25oneGZLeXPHx0uznPsT2HtsT25PpdNrVPgDo1DzRW+2/+f2vVwhPsjfJOMl4NBr978sAoIF5orck6cwVH2+RdLjNHABoa57oPSfpLNtfsH2ipF2SHmg7CwDaWPOFgZK8b/tqSb+VtCDptiQHmi8DgAbmejW0JA9JeqjxFgBoju/IAFAK0QNQCtEDUArRA1AK0QNQCtEDUArRA1AK0QNQCtEDUArRA1AK0QNQCtEDUArRA1AK0QNQCtEDUArRA1AK0QNQCtEDUArRA1AK0QNQCtEDUArRA1AK0QNQCtEDUArRA1AK0QNQCtEDUArRA1DKmtGzfZvtI7ZfXo9BANDSPGd6P5e0s/EOAFgXa0YvyZOS/r4OWwCguc6u6dneY3tiezKdTru6WQDoVGfRS7I3yTjJeDQadXWzANApHr0FUArRA1DKPE9ZuVvS7yWdbXvJ9nfbzwKANjasdUCS3esxBADWA3dvAZRC9ACUQvQAlEL0AJRC9ACUQvQAlEL0AJRC9ACUQvQAlEL0AJRC9ACUQvQAlEL0AJRC9ACUQvQAlEL0AJRC9ACUQvQAlEL0AJRC9ACUQvQAlEL0AJRC9ACUQvQAlEL0AJRC9ACUQvQAlLJm9Gyfaftx2wdtH7B9zXoMA4AWNsxxzPuSfphkv+3PSdpn+9EkrzTeBgCdW/NML8lbSfbP3n9X0kFJZ7QeBgAtfKJrerYXJW2V9Mwqn9tje2J7Mp1OO5oHAN2aO3q2PyvpXknXJnnno59PsjfJOMl4NBp1uREAOjNX9Gxv1HLw7kxyX9tJANDOPI/eWtKtkg4muan9JABoZ54zve2SrpS0w/YLs7dvNd4FAE2s+ZSVJE9J8jpsAYDm+I4MAKUQPQClED0ApRA9AKUQPQClED0ApRA9AKUQPQClED0ApRA9AKUQPQClED0ApRA9AKUQPQClED0ApRA9AKUQPQClED0ApRA9AKUQPQClED0ApRA9AKUQPQClED0ApRA9AKUQPQClED0ApawZPdsn2X7W9ou2D9i+cT2GAUALG+Y45l+SdiQ5anujpKds/ybJ0423AUDn1oxekkg6Ovtw4+wtLUcBQCtzXdOzvWD7BUlHJD2a5JlVjtlje2J7Mp1OO54JAN2YK3pJPkhynqQtkrbZPneVY/YmGScZj0ajjmcCQDc+0aO3Sf4h6QlJO1uMAYDW5nn0dmT71Nn7n5H0dUmvNt4FAE3M8+jt6ZJut72g5Uj+MsmDbWcBQBvzPHr7B0lb12ELADTHd2QAKIXoASiF6AEohegBKIXoASiF6AEohegBKIXoASiF6AEohegBKIXoASiF6AEohegBKIXoASiF6AEohegBKIXoASiF6AEohegBKIXoASiF6AEohegBKIXoASiF6AEohegBKIXoASiF6AEoZe7o2V6w/bztB1sOAoCWPsmZ3jWSDrYaAgDrYa7o2d4i6RJJt7SdAwBtzXumd7Ok6yT9++MOsL3H9sT2ZDqddrENADq3ZvRsXyrpSJJ9xzsuyd4k4yTj0WjU2UAA6NI8Z3rbJV1m+5CkeyTtsH1H01UA0Mia0UtyQ5ItSRYl7ZL0WJIrmi8DgAZ4nh6AUjZ8koOTPCHpiSZLAGAdcKYHoBSiB6AUogegFKIHoBSiB6AUogegFKIHoBSiB6AUogegFKIHoBSiB6AUogegFKIHoBSiB6AUogegFKIHoBSiB6AUogegFKIHoBSiB6AUogegFKIHoBSiB6AUogegFKIHoBSiB6AUogegFKIHoJQN8xxk+5CkdyV9IOn9JOOWowCglbmiN/O1JG83WwIA64C7twBKmTd6kfSI7X2296x2gO09tie2J9PptLuFANCheaO3Pcn5ki6WdJXtCz96QJK9ScZJxqPRqNORANCVuaKX5PDsxyOS7pe0reUoAGhlzejZ3mT7c8fel/RNSS+3HgYALczz6O1pku63fez4u5I83HQVADSyZvSSvCHpS+uwBQCa4ykrAEohegBKIXoASiF6AEohegBKIXoASiF6AEohegBKIXoASiF6AEohegBKIXoASnGS7m/Unkr6cwc3tVnSkF6Xgz3HN7Q90vA2sef4utrz+SSr/m/GTaLXFduTIb3yGnuOb2h7pOFtYs/xrcce7t4CKIXoAShl6NHb2/eAj2DP8Q1tjzS8Tew5vuZ7Bn1NDwC6NvQzPQDoFNEDUMogo2d7p+3XbL9u+/oB7LnN9hHbg3jpS9tn2n7c9kHbB2xf0/Oek2w/a/vF2Z4b+9xzjO0F28/bfrDvLZJk+5Dtl2y/YHsygD2n2v617Vdnf5a+0uOWs2e/Lsfe3rF9bZOvNbRrerYXJP1R0jckLUl6TtLuJK/0uOlCSUcl/SLJuX3tWLHndEmnJ9k/e03ifZK+3devkZdfH3RTkqO2N0p6StI1SZ7uY8+KXT+QNJZ0SpJL+9wy23NI0jjJIJ4MbPt2Sb9LcovtEyWdnOQfPc861oC/SPpyki6+yeFDhnimt03S60neSPKepHskXd7noCRPSvp7nxtWSvJWkv2z99+VdFDSGT3uSZKjsw83zt56/dfU9hZJl0i6pc8dQ2X7FEkXSrpVkpK8N4TgzVwk6U8tgicNM3pnSHpzxcdL6vEv9NDZXpS0VdIzPe9YsP2CpCOSHk3S6x5JN0u6TtK/e96xUiQ9Ynuf7T09b/mipKmkn80uAdxie1PPm47ZJenuVjc+xOh5lZ8b1n3wgbD9WUn3Sro2yTt9bknyQZLzJG2RtM12b5cBbF8q6UiSfX1t+Bjbk5wv6WJJV80um/Rlg6TzJf00yVZJ/5Q0hOvnJ0q6TNKvWn2NIUZvSdKZKz7eIulwT1sGa3bt7F5Jdya5r+89x8zuIj0haWePM7ZLumx2De0eSTts39HjHklSksOzH49Iul/Ll3L6siRpacUZ+a+1HMG+XSxpf5K/tfoCQ4zec5LOsv2FWfV3SXqg502DMnvg4FZJB5PcNIA9I9unzt7/jKSvS3q1rz1JbkiyJcmilv/8PJbkir72SJLtTbMHnTS7G/lNSb09GyDJXyW9afvs2U9dJKm3BwtX2K2Gd22l5VPcQUnyvu2rJf1W0oKk25Ic6HOT7bslfVXSZttLkn6c5NYeJ22XdKWkl2bX0STpR0ke6mnP6ZJunz3qdoKkXyYZxNNEBuQ0Sfcv/3ulDZLuSvJwv5P0fUl3zk4u3pD0nT7H2D5Zy8/a+F7TrzO0p6wAQEtDvHsLAM0QPQClED0ApRA9AKUQPQClED0ApRA9AKX8B1ZbnM85f5HfAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "plt.imshow(X, cmap=plt.cm.hot)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Defintion of the 2D filter mapping and the convolution funciton - these allow to slide a filter across the image block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def corr2d(X, K):  \n",
    "    \"\"\"Compute 2D cross-correlation.\"\"\"\n",
    "    h, w = K.shape\n",
    "    Y = torch.zeros((X.shape[0] - h + 1, X.shape[1] - w + 1))\n",
    "    for i in range(Y.shape[0]):\n",
    "        for j in range(Y.shape[1]):\n",
    "            Y[i, j] = (X[i:i + h, j:j + w] * K).sum()\n",
    "    return Y\n",
    "\n",
    "class Conv2D(nn.Module):\n",
    "    def __init__(self, kernel_size):\n",
    "        super().__init__()\n",
    "        self.weight = nn.Parameter(torch.rand(kernel_size))\n",
    "        self.bias = nn.Parameter(torch.zeros(1))\n",
    "\n",
    "    def forward(self, x):\n",
    "        return corr2d(x, self.weight) + self.bias"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "K = torch.tensor([[1.0, -1.0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAADGCAYAAAA5bUwDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAANTklEQVR4nO3df4hl5X3H8fena6TU2Eazq677o2tgKd2AIXIrVqU1jYJumq6F/qFNU9sKi1CLKf21RQgB/0lSWiRgIlsr1TbEfzTJIopVkxBa0Toaf2KNGxvqZhd3ayWxFLHab/+Ys811njszd/aee+8U3i8Y7jnnec55vnv2mf3MOefe2VQVkiQN+4l5FyBJWn8MB0lSw3CQJDUMB0lSw3CQJDVOmncBK9m4cWPt2LFj3mVIje888cS8S5BG+h+gqjLpcdZ1OOzYsYOFhYV5lyE1TsnE33vSVLzZ03G8rSRJahgOkqSG4SBJahgOkqSG4SBJahgOkqSG4SBJahgOkqSG4SBJahgOkqSG4SBJahgOkqTGROGQ5PQkDyZ5qXs9bYW+G5J8J8m9k4wpSZq+Sa8c9gEPV9VO4OFufTk3AC9MOJ4kaQYmDYc9wB3d8h3AlaM6JdkKfAy4bcLxJEkzMGk4nFlVRwC61zOW6Xcz8Kcs/j8UK0qyN8lCkoVjx45NWJ4k6USs+p/9JHkIOGtE043jDJDkV4GjVfVEkktW619V+4H9AIPBoMYZQ5LUr1XDoaouXa4tyatJNlfVkSSbgaMjul0E/FqS3cBPAj+d5O+r6rdOuGpJ0lRNelvpAHBNt3wN8PWlHarqz6tqa1XtAK4CvmEwSNL6Nmk4fBa4LMlLwGXdOknOTnLfpMVJkuZj1dtKK6mq14CPjth+GNg9Yvu3gG9NMqYkafr8hLQkqWE4SJIahoMkqWE4SJIahoMkqWE4SJIahoMkqWE4SJIahoMkqWE4SJIahoMkqWE4SJIahoMkqWE4SJIahoMkqWE4SJIahoMkqWE4SJIahoMkqWE4SJIahoMkqWE4SJIahoMkqWE4SJIaE4VDktOTPJjkpe71tBF9tiX5ZpIXkjyf5IZJxpQkTd+kVw77gIeraifwcLe+1NvAH1XVzwMXAL+fZNeE40qSpmjScNgD3NEt3wFcubRDVR2pqie75TeAF4AtE44rSZqiScPhzKo6AoshAJyxUuckO4APA49NOK4kaYpOWq1DkoeAs0Y03biWgZK8F7gb+FRV/WiFfnuBvQDbt29fyxCSpJ6sGg5VdelybUleTbK5qo4k2QwcXabfe1gMhi9X1T2rjLcf2A8wGAxqtfokSf2b9LbSAeCabvka4OtLOyQJ8DfAC1X1VxOOJ0magUnD4bPAZUleAi7r1klydpL7uj4XAZ8EfiXJU93X7gnHlSRN0aq3lVZSVa8BHx2x/TCwu1v+RyCTjCNJmi0/IS1JahgOkqSG4SBJahgOkqSG4SBJahgOkqSG4SBJahgOkqSG4SBJahgOkqSG4SBJahgOkqSG4SBJahgOkqSG4SBJahgOkqSG4SBJahgOkqSG4SBJahgOkqSG4SBJahgOkqSG4SBJahgOkqSG4SBJavQSDkkuT/JikoNJ9o1oT5IvdO3PJDmvj3ElSdMxcTgk2QDcAlwB7AKuTrJrSbcrgJ3d117gS5OOK0manj6uHM4HDlbVy1X1FnAXsGdJnz3AnbXoUeB9STb3MLYkaQr6CIctwCtD64e6bWvtA0CSvUkWkiwcO3ash/IkSWvVRzhkxLY6gT6LG6v2V9WgqgabNm2auDhJ0tr1EQ6HgG1D61uBwyfQR5K0TvQRDo8DO5Ock+Rk4CrgwJI+B4Df7t61dAHww6o60sPYkqQpOGnSA1TV20muBx4ANgC3V9XzSa7r2m8F7gN2AweB/wJ+d9JxJUnTk6qRt/7XhcFgUAsLC/MuQ2qcklGP0aT5exN4p2riCeonpCVJDcNBktQwHCRJDcNBktQwHCRJDcNBktQwHCRJDcNBktQwHCRJDcNBktQwHCRJDcNBktQwHCRJDcNBktQwHCRJDcNBktQwHCRJDcNBktQwHCRJDcNBktQwHCRJDcNBktQwHCRJDcNBktToJRySXJ7kxSQHk+wb0f6JJM90X48k+VAf40qSpmPicEiyAbgFuALYBVydZNeSbv8K/HJVnQvcBOyfdFxJ0vT0ceVwPnCwql6uqreAu4A9wx2q6pGqer1bfRTY2sO4kqQp6SMctgCvDK0f6rYt51rg/h7GlSRNyUk9HCMjttXIjslHWAyHi5c9WLIX2Auwffv2HsqTJK1VH1cOh4BtQ+tbgcNLOyU5F7gN2FNVry13sKraX1WDqhps2rSph/IkSWvVRzg8DuxMck6Sk4GrgAPDHZJsB+4BPllV3+1hTEnSFE18W6mq3k5yPfAAsAG4vaqeT3Jd134r8Gng/cAXkwC8XVWDSceWJE1HqkY+HlgXBoNBLSwszLsMqXFKRj1qk+bvTeCdqoknqJ+QliQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1egmHJJcneTHJwST7Vuj3C0neSfIbfYwrSZqOicMhyQbgFuAKYBdwdZJdy/T7HPDApGNKkqarjyuH84GDVfVyVb0F3AXsGdHvD4C7gaM9jClJmqI+wmEL8MrQ+qFu2/9JsgX4deDW1Q6WZG+ShSQLx44d66E8SdJa9REOGbGtlqzfDPxZVb2z2sGqan9VDapqsGnTph7KkySt1Uk9HOMQsG1ofStweEmfAXBXEoCNwO4kb1fV13oYX5LUsz7C4XFgZ5JzgB8AVwG/Odyhqs45vpzkb4F7DQZJWr8mDoeqejvJ9Sy+C2kDcHtVPZ/kuq591ecMkqT1JVVLHw+sH4PBoBYWFuZdhtQ4JaMetUnz9ybwTtXEE9RPSEuSGoaDJKlhOEiSGoaDJKlhOEiSGoaDJKlhOEiSGoaDJKlhOEiSGoaDJKlhOEiSGuv6dysleQN4cd51rGIj8O/zLmIM1tkv6+yXdfbn56rq1EkP0sev7J6mF6tqMO8iVpJkYb3XCNbZN+vsl3X2J0kvv63U20qSpIbhIElqrPdw2D/vAsbw/6FGsM6+WWe/rLM/vdS4rh9IS5LmY71fOUiS5sBwkCQ15hoOSU5P8mCSl7rX05bp9/0kzyZ5avhtWuPuP4s6k2xL8s0kLyR5PskNQ22fSfKDrv6nkuzuub7Lk7yY5GCSfSPak+QLXfszSc4bd98Z1/mJrr5nkjyS5ENDbSPnwBxqvCTJD4f+Lj897r4zrvNPhmp8Lsk7SU7v2mZyLruxbk9yNMlzy7TPfW6OUePc5+WYdfY7N6tqbl/A54F93fI+4HPL9Ps+sPFE959FncBm4Lxu+VTgu8Cubv0zwB9PqbYNwPeADwAnA08fH3eoz27gfiDABcBj4+474zovBE7rlq84XudKc2AONV4C3Hsi+86yziX9Pw58Y5bncmisXwLOA55bpn09zM3VapzrvFxDnb3OzXnfVtoD3NEt3wFcOeP9exunqo5U1ZPd8hvAC8CWKdUz7HzgYFW9XFVvAXd19Q7bA9xZix4F3pdk85j7zqzOqnqkql7vVh8Ftk6plhOucUr7TrvOq4GvTKmWFVXVt4H/WKHL3OfmajWug3l5vI7VzuVyTuhczjsczqyqI7D4jytwxjL9CviHJE8k2XsC+8+qTgCS7AA+DDw2tPn67rL09p5vf20BXhlaP0QbSsv1GWffvqx1rGtZ/InyuOXmQJ/GrfEXkzyd5P4kH1zjvn0Ye6wkPwVcDtw9tHkW53Jc62FursU85uVa9DY3p/7rM5I8BJw1ounGNRzmoqo6nOQM4MEk/9KlaG96qpMk72XxG/FTVfWjbvOXgJtYnEg3AX8J/N6JV/vuIUdsW/r+5OX6jLNvX8YeK8lHWPwmvHho89TnwJg1Pgn8bFX9ZxafHX0N2Dnmvn1Zy1gfB/6pqoZ/4pzFuRzXepibY5njvBxXr3Nz6uFQVZcu15bk1SSbq+pIdyl5dJljHO5ejyb5KouXSd8Gxtp/VnUmeQ+LwfDlqrpn6NivDvX5a+DeE61zhEPAtqH1rcDhMfucPMa+fRmnTpKcC9wGXFFVrx3fvsIcmGmNQ4FPVd2X5ItJNo6z7yzrHHIVS24pzehcjms9zM1VzXlejqX3uTmLBykrPGD5C979oPfzI/qcApw6tPwIcPm4+8+wzgB3AjePaNs8tPyHwF091nYS8DJwDj9+2PTBJX0+xrsf+v3zuPvOuM7twEHgwnHnwBxqPIsff3j0fODfuvO6rs5l1+9nWLxHfcqsz+WSOnaw/EPUuc/NMWqc67xcQ529zs2p/SHG/IO+H3gYeKl7Pb3bfjZwX7f8ge4P8zTwPHDjavvPqc6LWbxUewZ4qvva3bX9HfBs13aAobDoqb7dLL476nvHzw9wHXBdtxzglq79WWCw0r5T/Pterc7bgNeHzt/CanNgDjVe39XwNIsPJy9cad951dmt/w5LfhCZ5bnsxvsKcAT4bxZ/gr12vc3NMWqc+7wcs85e56a/PkOS1Jj3u5UkSeuQ4SBJahgOkqSG4SBJahgOkqSG4SBJahgOkqTG/wKKc/LZ66Qg9gAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(K, cmap=plt.cm.hot)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now apply the filter to the original image/data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.,  1.,  0.,  0.,  0., -1.,  0.],\n",
       "        [ 0.,  1.,  0.,  0.,  0., -1.,  0.],\n",
       "        [ 0.,  1.,  0.,  0.,  0., -1.,  0.],\n",
       "        [ 0.,  1.,  0.,  0.,  0., -1.,  0.],\n",
       "        [ 0.,  1.,  0.,  0.,  0., -1.,  0.],\n",
       "        [ 0.,  1.,  0.,  0.,  0., -1.,  0.]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y = corr2d(X, K)\n",
    "Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARkAAAD4CAYAAADYf5KEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAKaElEQVR4nO3d72tdhR3H8c9nSSWuKrKZSWnK6kAEEWZdKIyCbJ2TOkX3UEEfjEGezFHZQHRPhv+A+GQMStvNYbWIWhDxV0GLE+aPtOq0to7SdRjqSEREK3RS/exBT1nU2Ny6+805Ob5fEJqbXG4+aPvOueeEXCcRAFT5RtsDAPQbkQFQisgAKEVkAJQiMgBKjVY86AVjztqVFY/8FV30g7YXfNY/97a9AGfolffaXvBF677V9oL/OfKR9O7xeKHPlURm7Upp+pqKR/6K7ptue8Fn3bzg/wt02ModbS/4oi79G5t84ss/x9MlAKWIDIBSRAZAKSIDoBSRAVCKyAAoRWQAlCIyAEoRGQCliAyAUkQGQCkiA6AUkQFQaqDI2N5k+y3bh2zfUT0KQH8sGhnbI5L+IOkaSZdKusn2pdXDAPTDIEcy6yUdSnI4yceSdkq6oXYWgL4YJDKrJb097/ZM87HPsD1le9r29NzxYc0DsNwNEpmFfo3bF14RLsmWJJNJJsfH/v9hAPphkMjMSFoz7/aEpKM1cwD0zSCReVnSxbYvsn2WpBslPVo7C0BfLPqLxJOcsH2rpKckjUjanmR/+TIAvTDQqxUkeVzS48VbAPQQP/ELoBSRAVCKyAAoRWQAlCIyAEoRGQCliAyAUkQGQCkiA6AUkQFQisgAKEVkAJQiMgBKERkApYgMgFJEBkApIgOgFJEBUIrIAChFZACUIjIAShEZAKWIDIBSRAZAKSIDoBSRAVCKyAAoRWQAlFo0Mra32561/cZSDALQL4McyfxZ0qbiHQB6atHIJHlO0ntLsAVADw3tnIztKdvTtqfnjg/rUQEsd0OLTJItSSaTTI6PDetRASx3XF0CUIrIACg1yCXsByT9TdIltmds/7J+FoC+GF3sDkluWoohAPqJp0sAShEZAKWIDIBSRAZAKSIDoBSRAVCKyAAoRWQAlCIyAEoRGQCliAyAUkQGQCkiA6AUkQFQisgAKEVkAJQiMgBKERkApYgMgFJEBkApIgOgFJEBUIrIAChFZACUIjIAShEZAKWIDIBSi0bG9hrbz9o+YHu/7c1LMQxAP4wOcJ8Tkn6bZJ/tcyXttb07yZvF2wD0wKJHMkneSbKvef9DSQckra4eBqAfzuicjO21ktZJenGBz03ZnrY9PXd8SOsALHsDR8b2OZIelnRbkg8+//kkW5JMJpkcHxvmRADL2UCRsb1CJwOzI8kjtZMA9MkgV5csaZukA0nurp8EoE8GOZLZIOkWSRttv9q8/ax4F4CeWPQSdpLnJXkJtgDoIX7iF0ApIgOgFJEBUIrIAChFZACUIjIAShEZAKWIDIBSRAZAKSIDoBSRAVCKyAAoRWQAlCIyAEoRGQCliAyAUkQGQCkiA6AUkQFQisgAKEVkAJQiMgBKERkApYgMgFJEBkApIgOgFJEBUGrRyNges/2S7dds77d911IMA9APowPc5z+SNiY5ZnuFpOdtP5HkheJtAHpg0cgkiaRjzc0VzVsqRwHoj4HOydgesf2qpFlJu5O8uMB9pmxP256eOz7klQCWrYEik+STJJdLmpC03vZlC9xnS5LJJJPjY0NeCWDZOqOrS0nel7RH0qaKMQD6Z5CrS+O2z2/eP1vSVZIOFu8C0BODXF1aJele2yM6GaUHkzxWOwtAXwxydenvktYtwRYAPcRP/AIoRWQAlCIyAEoRGQCliAyAUkQGQCkiA6AUkQFQisgAKEVkAJQiMgBKERkApYgMgFJEBkApIgOgFJEBUIrIAChFZACUIjIAShEZAKWIDIBSRAZAKSIDoBSRAVCKyAAoRWQAlCIyAEoNHBnbI7Zfsf1Y5SAA/XImRzKbJR2oGgKgnwaKjO0JSddK2lo7B0DfDHokc4+k2yV9+mV3sD1le9r29NzxYUwD0AeLRsb2dZJmk+w93f2SbEkymWRyfGxo+wAsc4McyWyQdL3tI5J2Stpo+77SVQB6Y9HIJLkzyUSStZJulPRMkpvLlwHoBX5OBkCp0TO5c5I9kvaULAHQSxzJAChFZACUIjIAShEZAKWIDIBSRAZAKSIDoBSRAVCKyAAoRWQAlCIyAEoRGQCliAyAUkQGQCkiA6AUkQFQisgAKEVkAJQiMgBKERkApYgMgFJEBkApIgOgFJEBUIrIAChFZACUIjIAShEZAKVGB7mT7SOSPpT0iaQTSSYrRwHoj4Ei0/hxknfLlgDoJZ4uASg1aGQi6Wnbe21PLXQH21O2p21Pzx0f3kAAy9ugT5c2JDlq+zuSdts+mOS5+XdIskXSFkma/LYz5J0AlqmBjmSSHG3+nJW0S9L6ylEA+mPRyNheafvcU+9LulrSG9XDAPTDIE+XLpS0y/ap+9+f5MnSVQB6Y9HIJDks6ftLsAVAD3EJG0ApIgOgFJEBUIrIAChFZACUIjIAShEZAKWIDIBSRAZAKSIDoBSRAVCKyAAo5WT4v1/K9pykfw3hoS6Q1KXfK8ye0+vaHql7m/q657tJxhf6RElkhsX2dJdeGYE9p9e1PVL3Nn0d9/B0CUApIgOgVNcjs6XtAZ/DntPr2h6pe5u+dns6fU4GwPLX9SMZAMsckQFQqpORsb3J9lu2D9m+owN7ttuetd2Jl4Kxvcb2s7YP2N5ve3PLe8Zsv2T7tWbPXW3uOcX2iO1XbD/W9hZJsn3E9uu2X7U93YE959t+yPbB5u/SD0u+TtfOydgekfQPST+VNCPpZUk3JXmzxU1XSjom6S9JLmtrx7w9qyStSrKveU2svZJ+3tZ/I598vZyVSY7ZXiHpeUmbk7zQxp55u34jaVLSeUmua3NLs+eIpMkknfhhPNv3Svprkq22z5L0zSTvD/vrdPFIZr2kQ0kOJ/lY0k5JN7Q5qHlJ3vfa3DBfkneS7Gve/1DSAUmrW9yTJMeamyuat1a/e9mekHStpK1t7ugq2+dJulLSNklK8nFFYKRuRma1pLfn3Z5Ri/+Aus72WknrJL3Y8o4R269KmpW0O0mreyTdI+l2SZ+2vGO+SHra9l7bUy1v+Z6kOUl/ap5Sbm1eIXbouhgZL/Cxbj2n6wjb50h6WNJtST5oc0uST5JcLmlC0nrbrT2ttH2dpNkke9va8CU2JLlC0jWSftU8DW/LqKQrJP0xyTpJH0kqOf/ZxcjMSFoz7/aEpKMtbems5tzHw5J2JHmk7T2nNIfceyRtanHGBknXN+dAdkraaPu+FvdIkpIcbf6clbRLJ08NtGVG0sy8I86HdDI6Q9fFyLws6WLbFzUno26U9GjLmzqlOdG6TdKBJHd3YM+47fOb98+WdJWkg23tSXJnkokka3Xy788zSW5ua48k2V7ZnKRX87TkakmtXa1M8m9Jb9u+pPnQTySVXDhY9LWwl1qSE7ZvlfSUpBFJ25Psb3OT7Qck/UjSBbZnJP0+ybYWJ22QdIuk15vzIJL0uySPt7RnlaR7myuD35D0YJJOXDbukAsl7Tr5/UGjku5P8mS7k/RrSTuab+aHJf2i4ot07hI2gH7p4tMlAD1CZACUIjIAShEZAKWIDIBSRAZAKSIDoNR/AdRyevKhTBBKAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(Y, cmap=plt.cm.hot)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the above example we used a static kernel. But in large networks this will not be possible. In CNN training we can use the training data to learn a kernel (that's one beauty of learning via multiple iteration). We can start from a random kernel state and then learn it through the training phase. Now we can explore a kernel can be leartned. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 2, loss 1.373\n",
      "epoch 4, loss 0.240\n",
      "epoch 6, loss 0.044\n",
      "epoch 8, loss 0.009\n",
      "epoch 10, loss 0.002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/payam/anaconda3/lib/python3.8/site-packages/torch/nn/modules/lazy.py:178: UserWarning: Lazy modules are a new feature under heavy development so changes to the API or functionality can happen at any moment.\n",
      "  warnings.warn('Lazy modules are a new feature under heavy development '\n"
     ]
    }
   ],
   "source": [
    "# Construct a two-dimensional convolutional layer with 1 output channel and a\n",
    "# kernel of shape (1, 2). For the sake of simplicity, we ignore the bias here\n",
    "# adapted from https://d2l.ai/chapter_convolutional-neural-networks/conv-layer.html\n",
    "\n",
    "conv2d = nn.LazyConv2d(1, kernel_size=(1, 2), bias=False)\n",
    "\n",
    "# The two-dimensional convolutional layer uses four-dimensional input and\n",
    "# output in the format of (example, channel, height, width), where the batch\n",
    "# size (number of examples in the batch) and the number of channels are both 1\n",
    "X = X.reshape((1, 1, 6, 8))\n",
    "Y = Y.reshape((1, 1, 6, 7))\n",
    "lr = 3e-2  # Learning rate\n",
    "\n",
    "for i in range(10):\n",
    "    Y_hat = conv2d(X)\n",
    "    l = (Y_hat - Y) ** 2\n",
    "    conv2d.zero_grad()\n",
    "    l.sum().backward()\n",
    "    # Update the kernel\n",
    "    conv2d.weight.data[:] -= lr * conv2d.weight.grad\n",
    "    if (i + 1) % 2 == 0:\n",
    "        print(f'epoch {i + 1}, loss {l.sum():.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[[1., 1., 0., 0., 0., 0., 1., 1.],\n",
       "          [1., 1., 0., 0., 0., 0., 1., 1.],\n",
       "          [1., 1., 0., 0., 0., 0., 1., 1.],\n",
       "          [1., 1., 0., 0., 0., 0., 1., 1.],\n",
       "          [1., 1., 0., 0., 0., 0., 1., 1.],\n",
       "          [1., 1., 0., 0., 0., 0., 1., 1.]]]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[[ 0.,  1.,  0.,  0.,  0., -1.,  0.],\n",
       "          [ 0.,  1.,  0.,  0.,  0., -1.,  0.],\n",
       "          [ 0.,  1.,  0.,  0.,  0., -1.,  0.],\n",
       "          [ 0.,  1.,  0.,  0.,  0., -1.,  0.],\n",
       "          [ 0.,  1.,  0.,  0.,  0., -1.,  0.],\n",
       "          [ 0.,  1.,  0.,  0.,  0., -1.,  0.]]]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[[ 0.9975, -0.9905]]]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conv2d.weight.data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is pretty cool! \n",
    "\n",
    "In a few iterations we almost learned the same edge detection kernel that we have hard coded intially [1.0, -1.0] (see above). \n",
    "\n",
    "Excercise: Try to change the learning rate (lr) and also the number of iterations (for i in range(10)) (e.g. change to 20) and see how the results change."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.12 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12 (default, Oct 12 2021, 06:23:56) \n[Clang 10.0.0 ]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "d2d9da60ce1b18cfa0060049e333eb5cea5df408c451157604e0a4dc4cdbeaeb"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}