NaiveBayes1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Andrew Carr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import time\n",
    "\n",
    "from scipy import stats\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "from sklearn import datasets\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### build normally distributed data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# parameters\n",
    "mean, std_dev= 0,1\n",
    "n = 1000\n",
    "classes = [0,1]\n",
    "\n",
    "# choose 3 features\n",
    "x = np.array([[i,j,k] for (i,j,k) in \n",
    "              zip(np.random.normal(loc=mean, scale=std_dev, size=n), \n",
    "                  np.random.normal(loc=mean, scale=std_dev, size=n), \n",
    "                  np.random.normal(loc=mean, scale=std_dev, size=n))\n",
    "             ])\n",
    "\n",
    "\n",
    "# get classes for these features\n",
    "y = np.array([c for c in np.random.choice(classes, size=n)])\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(x,y, train_size=0.7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load cancer data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = datasets.load_breast_cancer()\n",
    "train_x, test_x, train_y, test_y = train_test_split(data.data, data.target, train_size=0.7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### personal naive bayes classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class NB(object):\n",
    "    \"\"\"Naive Bayes Classifier Object\"\"\"\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def fit(self, x,y):\n",
    "        self.train_x = x\n",
    "        self.train_y = y\n",
    "        self.num_samples,self.num_features = self.train_x.shape\n",
    "        classes = list(set(y))\n",
    "        self.stats = []\n",
    "        # for both classes\n",
    "        for c in classes:\n",
    "            features = self.train_x[self.train_y == c]\n",
    "            for feature_index in range(self.num_features):\n",
    "                # select the column of features\n",
    "                feature = features[:,feature_index]\n",
    "                \n",
    "                # gather statistics\n",
    "                mean = np.mean(feature)\n",
    "                std_dev = np.sqrt(np.var(feature))\n",
    "                \n",
    "                # build and save a normal based on the data associated with the class\n",
    "                self.stats.append(stats.norm(loc=mean,scale=std_dev))\n",
    "                \n",
    "        \n",
    "    def predict(self, x):\n",
    "        self.test_x = x\n",
    "        self.probs = []\n",
    "        self.prediction = []\n",
    "        # for each data point\n",
    "        for instance in self.test_x:\n",
    "            probability_0 = 0\n",
    "            probability_1 = 0\n",
    "            for feature_index in range(len(instance)):\n",
    "                # get the normal associated with the current feature and evaluate for class 0\n",
    "                probability_0 += np.log(self.stats[feature_index].pdf(instance[feature_index]))\n",
    "                \n",
    "                # get the normal associated with the current feature and evaluate for class 1\n",
    "                probability_1 += np.log(self.stats[feature_index+self.num_features].pdf(instance[feature_index]))\n",
    "            \n",
    "            # determine the correct class, not worrying about p(class) since we have no prior knowledge, \n",
    "            # and so assuming uniform gives no bias and has good accuracy\n",
    "            self.prediction.append(np.argmax([probability_0, probability_1]))\n",
    "            self.probs.append([probability_0, probability_1])\n",
    "        return self.prediction\n",
    "\n",
    "        \n",
    "    def accuracy(self, y_true,y_pred):\n",
    "        return accuracy_score(y_true, y_pred)\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classifier on random data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test accuracy on random data 0.4633333333333333\n"
     ]
    }
   ],
   "source": [
    "clf = NB()\n",
    "\n",
    "clf.fit(x_train,y_train)\n",
    "\n",
    "y_pred = clf.predict(x_test)\n",
    "\n",
    "print(\"test accuracy on random data {}\".format(clf.accuracy(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classifier on cancer data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training time 0.058775901794433594\n",
      "test accuracy on cancer data 0.9532163742690059\n"
     ]
    }
   ],
   "source": [
    "clf = NB()\n",
    "\n",
    "start = time.time()\n",
    "clf.fit(train_x, train_y)\n",
    "print(\"training time {}\".format(time.time() - start))\n",
    "\n",
    "pred_y = clf.predict(test_x)\n",
    "\n",
    "print(\"test accuracy on cancer data {}\".format(clf.accuracy(test_y, pred_y)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sklearn Classification on cancer data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training time 0.0026929378509521484\n",
      "test accuracy on cancer data 0.9473684210526315\n"
     ]
    }
   ],
   "source": [
    "clf = GaussianNB()\n",
    "\n",
    "start = time.time()\n",
    "clf.fit(train_x, train_y)\n",
    "print(\"training time {}\".format(time.time() - start))\n",
    "\n",
    "pred_y = clf.predict(test_x)\n",
    "\n",
    "print(\"test accuracy on cancer data {}\".format(accuracy_score(test_y, pred_y)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conclusion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As can be seen above, my naive bayes classifier is competitive with the industry standard in accuracy, however it is signifiantly slower. This is not surprising since my code is obviously inefficent and naive. However, it is very exciting that it is comparable in accuracy and was quite simple to implement."
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}