jax/03_logistic_neurons.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Neurons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import jax\n",
    "from sklearn.metrics import accuracy_score\n",
    "import matplotlib.pyplot as plt\n",
    "from utils import make_classification, draw_decision_boundary, sigmoid\n",
    "\n",
    "jnp = jax.numpy\n",
    "\n",
    "plt.style.use('ggplot')\n",
    "plt.rc('figure', figsize=(8, 6))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Activation of a logistic neuron:\n",
    "\n",
    "## $$ z = \\sum_{i \\in L} x_{i}w_{i} + b$$ \n",
    "\n",
    "## Predicted output:\n",
    "\n",
    "## $$ h = \\frac{1}{1 + e^{-z}} $$\n",
    "\n",
    "## Log Loss:\n",
    "## $$ E = -\\sum_{i=1}^{n}\\Big(y^i \\log(h^{i}) + (1 - y^i) \\log(1 - h^{i})\\Big) $$\n",
    "\n",
    "# Logistic Neuron in NumPy:\n",
    "\n",
    "## Step 1: Make dummy data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "X, Y = make_classification()\n",
    "W = np.random.rand(2, 1)\n",
    "B = np.random.rand(1,)\n",
    "\n",
    "plt.scatter(*X.T, c=Y.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "draw_decision_boundary(W.ravel().tolist() + [B[0]], X, Y.ravel())\n",
    "plt.ylim(X[:, 1].min() - 0.1, X[:, 1].max() + 0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Get activation and prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# activation\n",
    "Z = np.dot(X, W) + B\n",
    "\n",
    "# prediction\n",
    "Y_pred = sigmoid(Z)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Gradient Descent for Logistic Regression\n",
    "\n",
    "## $$ \\mathbf{W}_{t + 1} = \\mathbf{W}_{t} - \\alpha(\\mathbf{X}^T(\\sigma(\\mathbf{XW}) - Y))$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using this gradient to train neuron with NumPy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(X, weights, bias=None):\n",
    "    z = np.dot(X, weights)\n",
    "    if bias is not None:\n",
    "        z += bias\n",
    "    return sigmoid(z)\n",
    "\n",
    "def train(X, Y, weights, alpha=0.001):\n",
    "    y_hat = predict(X, weights)\n",
    "    weights -= alpha * np.dot(X.T, y_hat - Y)\n",
    "    return weights\n",
    "\n",
    "def loss(y1, y2):\n",
    "    return (0.5 * ((y1 - y2) ** 2)).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "X, Y = make_classification()\n",
    "W = np.random.rand(2, 1)\n",
    "B = np.random.rand(1,)\n",
    "\n",
    "\n",
    "for i in range(10000):\n",
    "    y_hat = predict(X, W)\n",
    "    W = train(X, Y, W)\n",
    "    if i % 1000 == 0:\n",
    "        print(\"Loss: \", loss(Y, y_hat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "draw_decision_boundary(W.ravel().tolist() + [B[0]], X, Y.ravel())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression with Jax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, Y = make_classification()\n",
    "X, Y = map(jnp.array, (X, Y.ravel()))\n",
    "\n",
    "# Initialize weights and biases\n",
    "W = jnp.array(np.random.rand(2))\n",
    "b = np.random.rand()\n",
    "\n",
    "draw_decision_boundary(W.ravel().tolist() + [b], X, Y)\n",
    "# plt.ylim(X[:, 1].min() - 0.1, X[:, 1].max() + 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + jnp.exp(-x))\n",
    "\n",
    "def predict(X, W, b):\n",
    "    return sigmoid(jnp.dot(X, W) + b)\n",
    "\n",
    "def loss(W, b, X, Y):\n",
    "    H = predict(X, W, b)\n",
    "    l = Y * H + (1 - Y) * (1 - H)\n",
    "    return -jnp.sum(jnp.log(l))\n",
    "\n",
    "alpha = 1e-3\n",
    "w_grad = jax.grad(loss, 0)\n",
    "b_grad = jax.grad(loss, 1)\n",
    "\n",
    "for i in range(10000):\n",
    "    W -= alpha * w_grad(W, b, X, Y)\n",
    "    b -= alpha * b_grad(W, b, X, Y)\n",
    "    if i % 1000 == 0:\n",
    "        print(loss(W, b, X, Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_decision_boundary(W.tolist() + [b], X, Y)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}