13 Class Exercise.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 103/10000 [00:00<00:09, 1026.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training the network...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10000/10000 [00:09<00:00, 1040.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1], [0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1], [1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1], [1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "#Based on above predict method calculate probability of the following 4 shapes:\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "import math\n",
    "\n",
    "def sigmoid(t):\n",
    "    \"\"\"a step function made continous in order to use calculus (differential)\"\"\"\n",
    "    return 1 / (1 + math.exp(-t)) # t=0 -> 1/2, t=100 -> ~ 1, t=-100 -> ~0\n",
    "\n",
    "def neuron_output(weights, inputs):\n",
    "    \"\"\"reduces all values from input array and weights array to a value between 0 to 1\"\"\"\n",
    "    return sigmoid(dot(weights, inputs)) \n",
    "\n",
    "def dot(v, w):\n",
    "    \"\"\"v_1 * w_1 + ... + v_n * w_n\"\"\"\n",
    "    return sum(v_i * w_i for v_i, w_i in zip(v, w))\n",
    "\n",
    "raw_digits = [\n",
    "        \"\"\"@@@@@\n",
    "           @...@\n",
    "           @...@\n",
    "           @...@\n",
    "           @@@@@\"\"\",\n",
    "        \"\"\"..@..\n",
    "           ..@..\n",
    "           ..@..\n",
    "           ..@..\n",
    "           ..@..\"\"\",\n",
    "        \"\"\"@@@@@\n",
    "           ....@\n",
    "           @@@@@\n",
    "           @....\n",
    "           @@@@@\"\"\",\n",
    "        \"\"\"@@@@@\n",
    "           ....@\n",
    "           @@@@@\n",
    "           ....@\n",
    "           @@@@@\"\"\",\n",
    "        \"\"\"@...@\n",
    "           @...@\n",
    "           @@@@@\n",
    "           ....@\n",
    "           ....@\"\"\",\n",
    "        \"\"\"@@@@@\n",
    "           @....\n",
    "           @@@@@\n",
    "           ....@\n",
    "           @@@@@\"\"\",\n",
    "        \"\"\"@@@@@\n",
    "           @....\n",
    "           @@@@@\n",
    "           @...@\n",
    "           @@@@@\"\"\",\n",
    "        \"\"\"@@@@@\n",
    "           ....@\n",
    "           ....@\n",
    "           ....@\n",
    "           ....@\"\"\",\n",
    "        \"\"\"@@@@@\n",
    "           @...@\n",
    "           @@@@@\n",
    "           @...@\n",
    "           @@@@@\"\"\",\n",
    "        \"\"\"@@@@@\n",
    "           @...@\n",
    "           @@@@@\n",
    "           ....@\n",
    "           @@@@@\"\"\"]\n",
    "\n",
    "def make_digit(raw_digit):\n",
    "    \"\"\"transform digit set to using zeros instead of dots\"\"\"\n",
    "    return [1 if c == '@' else 0\n",
    "            for row in raw_digit.split(\"\\n\")\n",
    "            for c in row.strip()]\n",
    "\n",
    "\n",
    "inputs = [make_digit(raw_digit) for raw_digit in raw_digits]\n",
    "\n",
    "targets = [[1 if i == j else 0 for i in range(10)] for j in range(10)]\n",
    "\n",
    "def feed_forward(neural_network, input_vector):\n",
    "    \"\"\"takes in a neural network\n",
    "    (represented as a list of lists(non-input layers) of lists(neurons) of weights)\n",
    "    and returns the output from forward-propagating the input\"\"\"\n",
    "    outputs = []\n",
    "    # process one layer at a time\n",
    "    for layer in neural_network:\n",
    "        input_with_bias = input_vector + [1]               # add a bias input\n",
    "        output = [neuron_output(neuron, input_with_bias)   # compute the output\n",
    "            for neuron in layer]                           # for each neuron\n",
    "        outputs.append(output)                             # and remember it\n",
    "        \n",
    "        # then the input to the next layer is the output of this one\n",
    "        input_vector = output\n",
    "    return outputs\n",
    "\n",
    "def backpropagate(network, input_vector, targets):\n",
    "    \"\"\"\n",
    "    1. Run feed_forward on an input vector to produce the outputs of all the neurons\n",
    "    in the network.\n",
    "    2. This results in an error for each output neuron—the difference between its out‐\n",
    "    put and its target.\n",
    "    3. Compute the gradient of this error as a function of the neuron’s weights, and\n",
    "    adjust its weights in the direction that most decreases the error.\n",
    "    4. “Propagate” these output errors backward to infer errors for the hidden layer.\n",
    "    5. Compute the gradients of these errors and adjust the hidden layer’s weights in the\n",
    "    same manner.\n",
    "    \"\"\"\n",
    "    # We assume a single hidden layer from the network given to feed_forward function\n",
    "    hidden_outputs, outputs = feed_forward(network, input_vector)\n",
    "    # the output * (1 - output) is from the derivative of sigmoid\n",
    "    output_deltas = [output * (1 - output) * (output - target)\n",
    "                     for output, target in zip(outputs, targets)]\n",
    "    # adjust weights for output layer, one neuron at a time\n",
    "    for i, output_neuron in enumerate(network[-1]):\n",
    "        # focus on the ith output layer neuron\n",
    "        for j, hidden_output in enumerate(hidden_outputs + [1]):\n",
    "            # adjust the jth weight based on both\n",
    "            # this neuron's delta and its jth input\n",
    "            output_neuron[j] -= output_deltas[i] * hidden_output\n",
    "    # back-propagate errors to hidden layer\n",
    "    hidden_deltas = [hidden_output * (1 - hidden_output) *\n",
    "                     dot(output_deltas, [n[i] for n in output_layer])\n",
    "                     for i, hidden_output in enumerate(hidden_outputs)]\n",
    "    # adjust weights for hidden layer, one neuron at a time\n",
    "    for i, hidden_neuron in enumerate(network[0]):\n",
    "        for j, input in enumerate(input_vector + [1]):\n",
    "            hidden_neuron[j] -= hidden_deltas[i] * input\n",
    "            \n",
    "def predict(in_put):\n",
    "    return feed_forward(network, in_put)[-1]\n",
    "\n",
    "\n",
    "\n",
    "random.seed(0)   # to get repeatable results\n",
    "input_size = 25  # each input is a vector of length 25 (5x5 \"pixels\")\n",
    "\n",
    "num_hidden = 5   # we'll have 5 neurons in the hidden layer\n",
    "output_size = 10 # we need 10 outputs for each input\n",
    "\n",
    "# each hidden neuron has one weight per input, plus a bias weight\n",
    "hidden_layer = [[random.random() for _ in range(input_size + 1)]\n",
    "                 for _ in range(num_hidden)]\n",
    "#print(hidden_layer)\n",
    "\n",
    "# each output neuron has one weight per hidden neuron, plus a bias weight\n",
    "output_layer = [[random.random() for _ in range(num_hidden + 1)]\n",
    "                 for _ in range(output_size)]\n",
    "#print(output_layer)\n",
    "\n",
    "# the network starts out with random weights, one hidden layer and one output layer\n",
    "network = [hidden_layer, output_layer]\n",
    "\n",
    "\n",
    "print('Training the network...')\n",
    "\n",
    "# 10,000 iterations seems enough to converge\n",
    "for _ in tqdm(range(10000)):\n",
    "    for input_vector, target_vector in zip(inputs, targets): # inputs is a matrix of 10x25 (ten digits by 25 pixels), target is a one-hot encoded matrix of 10x10 (10 digits by 10 indices where each row has only 1 one and 9 zeroes)\n",
    "        backpropagate(network, input_vector, target_vector)\n",
    "\n",
    "raw_test = [\n",
    " \"\"\"@@@@@\n",
    "    @...@\n",
    "    @..@@\n",
    "    @...@\n",
    "    @@@@@\"\"\",\n",
    " \"\"\".@@@@\n",
    "    ....@\n",
    "    @@.@@\n",
    "    @....\n",
    "    @@@@@\"\"\",\n",
    " \"\"\"@.@.@\n",
    "    @...@\n",
    "    @@@@@\n",
    "    ....@\n",
    "    ....@\"\"\",\n",
    " \"\"\"@@@@@\n",
    "    @....\n",
    "    @@.@@\n",
    "    @...@\n",
    "    @@@@@\"\"\"]\n",
    "\n",
    "raw_digits = [\n",
    "        \"\"\"@@@@@\n",
    "           @...@\n",
    "           @...@\n",
    "           @...@\n",
    "           @@@@@\"\"\",\n",
    "        \"\"\"..@..\n",
    "           ..@..\n",
    "           ..@..\n",
    "           ..@..\n",
    "           ..@..\"\"\",\n",
    "        \"\"\"@@@@@\n",
    "           ....@\n",
    "           @@@@@\n",
    "           @....\n",
    "           @@@@@\"\"\",\n",
    "        \"\"\"@@@@@\n",
    "           ....@\n",
    "           @@@@@\n",
    "           ....@\n",
    "           @@@@@\"\"\",\n",
    "        \"\"\"@...@\n",
    "           @...@\n",
    "           @@@@@\n",
    "           ....@\n",
    "           ....@\"\"\",\n",
    "        \"\"\"@@@@@\n",
    "           @....\n",
    "           @@@@@\n",
    "           ....@\n",
    "           @@@@@\"\"\",\n",
    "        \"\"\"@@@@@\n",
    "           @....\n",
    "           @@@@@\n",
    "           @...@\n",
    "           @@@@@\"\"\",\n",
    "        \"\"\"@@@@@\n",
    "           ....@\n",
    "           ....@\n",
    "           ....@\n",
    "           ....@\"\"\",\n",
    "        \"\"\"@@@@@\n",
    "           @...@\n",
    "           @@@@@\n",
    "           @...@\n",
    "           @@@@@\"\"\",\n",
    "        \"\"\"@@@@@\n",
    "           @...@\n",
    "           @@@@@\n",
    "           ....@\n",
    "           @@@@@\"\"\"]\n",
    "\n",
    "test_data = [make_digit(test_digit) for test_digit in raw_test]\n",
    "#inputs2 = [make_digit(raw_digit) for raw_digit in raw_digits]\n",
    "print(test_data)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}