ML - Data Cleaning and Processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "dataset = pd.read_csv(\"simple_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country</th>\n",
       "      <th>Age</th>\n",
       "      <th>Salary</th>\n",
       "      <th>Purchased</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>France</td>\n",
       "      <td>44.0</td>\n",
       "      <td>72000.0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Spain</td>\n",
       "      <td>27.0</td>\n",
       "      <td>48000.0</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Germany</td>\n",
       "      <td>30.0</td>\n",
       "      <td>54000.0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Spain</td>\n",
       "      <td>38.0</td>\n",
       "      <td>61000.0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Germany</td>\n",
       "      <td>40.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>France</td>\n",
       "      <td>35.0</td>\n",
       "      <td>58000.0</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Spain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>52000.0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>France</td>\n",
       "      <td>48.0</td>\n",
       "      <td>79000.0</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Germany</td>\n",
       "      <td>50.0</td>\n",
       "      <td>83000.0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>France</td>\n",
       "      <td>37.0</td>\n",
       "      <td>67000.0</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Country   Age   Salary Purchased\n",
       "0   France  44.0  72000.0        No\n",
       "1    Spain  27.0  48000.0       Yes\n",
       "2  Germany  30.0  54000.0        No\n",
       "3    Spain  38.0  61000.0        No\n",
       "4  Germany  40.0      NaN       Yes\n",
       "5   France  35.0  58000.0       Yes\n",
       "6    Spain   NaN  52000.0        No\n",
       "7   France  48.0  79000.0       Yes\n",
       "8  Germany  50.0  83000.0        No\n",
       "9   France  37.0  67000.0       Yes"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X = dataset.iloc[:, :-1].values #takes all columns except the last 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['France', 44.0, 72000.0],\n",
       "       ['Spain', 27.0, 48000.0],\n",
       "       ['Germany', 30.0, 54000.0],\n",
       "       ['Spain', 38.0, 61000.0],\n",
       "       ['Germany', 40.0, nan],\n",
       "       ['France', 35.0, 58000.0],\n",
       "       ['Spain', nan, 52000.0],\n",
       "       ['France', 48.0, 79000.0],\n",
       "       ['Germany', 50.0, 83000.0],\n",
       "       ['France', 37.0, 67000.0]], dtype=object)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X #would traditionally be called a matrix of features for ML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y = dataset.iloc[:,-1].values #takes the last column\n",
    "#This is our dependent variable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['No', 'Yes', 'No', 'No', 'Yes', 'Yes', 'No', 'Yes', 'No', 'Yes'], dtype=object)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Suppose we have missing values in our dataset - we need a way to deal with that. A majority of time is often spent cleaning the data and structuring it in a way that is suitable to run ML algorithms. The following will illustrate how to deal with missing code."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import Imputer # this is a class! - refresh OOP if this confuses you."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "imputer = Imputer(missing_values='NaN', strategy=\"mean\", axis=0)\n",
    "#replaces missing values which are represented as NaN with the mean\n",
    "#of the column. We know we are dealing with a column as the axis=0\n",
    "#if axis=1 then we would be dealing with rows. Makes sense for column\n",
    "#here as we are dealing with features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now we need to fit this imputer object to our data!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "imputer = imputer.fit(X[:, 1:3]) #gives us the imputed values\n",
    "#recall that you specify where you want the fit.\n",
    "#in this case we want the fit on the last 2 columns only (age, salary)\n",
    "#and we update the matrix by  \n",
    "X[:, 1:3] = imputer.transform(X[:, 1:3])\n",
    "#this line actually changes X with the imputed values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['France', 44.0, 72000.0],\n",
       "       ['Spain', 27.0, 48000.0],\n",
       "       ['Germany', 30.0, 54000.0],\n",
       "       ['Spain', 38.0, 61000.0],\n",
       "       ['Germany', 40.0, 63777.77777777778],\n",
       "       ['France', 35.0, 58000.0],\n",
       "       ['Spain', 38.77777777777778, 52000.0],\n",
       "       ['France', 48.0, 79000.0],\n",
       "       ['Germany', 50.0, 83000.0],\n",
       "       ['France', 37.0, 67000.0]], dtype=object)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now we are going to learn how to encode categorical data.\n",
    "\n",
    "Country, Purchased etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder #this is a class!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "labelencoder_x = LabelEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X[:, 0] = labelencoder_x.fit_transform(X[:, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 44.0, 72000.0],\n",
       "       [2, 27.0, 48000.0],\n",
       "       [1, 30.0, 54000.0],\n",
       "       [2, 38.0, 61000.0],\n",
       "       [1, 40.0, 63777.77777777778],\n",
       "       [0, 35.0, 58000.0],\n",
       "       [2, 38.77777777777778, 52000.0],\n",
       "       [0, 48.0, 79000.0],\n",
       "       [1, 50.0, 83000.0],\n",
       "       [0, 37.0, 67000.0]], dtype=object)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X #now you see how it is encoded."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note - we use the labelencoder before onehotencoder in order to turn strings into integers (i.e. 'Spain' --> 2). It is only after we have turned these categorical variables into numeric types that we can turn them into dummy variables with onehotencoder.\n",
    "\n",
    "Now, note how the encoding as some form of value in that the computer might think that 2 (spain) is higher in value than 0 (France). We need to correct this. If our category was small, medium, large, then we could use this form of numbering as one really is bigger than the other.\n",
    "\n",
    "The answer to solving this is Dummy Variables!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "onehotencoder = OneHotEncoder(categorical_features=[0])\n",
    "#we have to specify which columns contain the categorical stuff.\n",
    "#countries is in index 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X = onehotencoder.fit_transform(X).toarray()\n",
    "#no need to respecify where in X ^^ as we did that already with [0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  1.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          4.40000000e+01,   7.20000000e+04],\n",
       "       [  0.00000000e+00,   0.00000000e+00,   1.00000000e+00,\n",
       "          2.70000000e+01,   4.80000000e+04],\n",
       "       [  0.00000000e+00,   1.00000000e+00,   0.00000000e+00,\n",
       "          3.00000000e+01,   5.40000000e+04],\n",
       "       [  0.00000000e+00,   0.00000000e+00,   1.00000000e+00,\n",
       "          3.80000000e+01,   6.10000000e+04],\n",
       "       [  0.00000000e+00,   1.00000000e+00,   0.00000000e+00,\n",
       "          4.00000000e+01,   6.37777778e+04],\n",
       "       [  1.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          3.50000000e+01,   5.80000000e+04],\n",
       "       [  0.00000000e+00,   0.00000000e+00,   1.00000000e+00,\n",
       "          3.87777778e+01,   5.20000000e+04],\n",
       "       [  1.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          4.80000000e+01,   7.90000000e+04],\n",
       "       [  0.00000000e+00,   1.00000000e+00,   0.00000000e+00,\n",
       "          5.00000000e+01,   8.30000000e+04],\n",
       "       [  1.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          3.70000000e+01,   6.70000000e+04]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "V = pd.DataFrame(X)\n",
    "pd.set_option('precision',0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>44</td>\n",
       "      <td>72000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>48000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>54000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>38</td>\n",
       "      <td>61000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>63778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>58000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>39</td>\n",
       "      <td>52000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>79000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>83000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>37</td>\n",
       "      <td>67000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2   3      4\n",
       "0  1  0  0  44  72000\n",
       "1  0  0  1  27  48000\n",
       "2  0  1  0  30  54000\n",
       "3  0  0  1  38  61000\n",
       "4  0  1  0  40  63778\n",
       "5  1  0  0  35  58000\n",
       "6  0  0  1  39  52000\n",
       "7  1  0  0  48  79000\n",
       "8  0  1  0  50  83000\n",
       "9  1  0  0  37  67000"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "V #you can see from here that 3 coulumns were added for the dummies."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Column 0 corresponds to france, 1 to spain and 3 is germany.\n",
    "\n",
    "Thats all there is to encoding."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now we need to do the same for the depedent variable y. However, we dont need to onehotencode anything here as we are using ML and the algorithsm know that Y is categorical. Furthermore, there is a relational meaning between 0 and 1.\n",
    "\n",
    "As such, we just use label encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "labelencoder_y = LabelEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y = labelencoder_y.fit_transform(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, 0, 1, 1, 0, 1, 0, 1])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "0 is no and 1 is yes\n",
    "\n",
    "Also, note how we used the labelencoder above and NOT onehotencode. This is a soure of condusion sometimes.\n",
    "\n",
    "Labelencoder: String -> int/float (if binary then we can stop here), if not binary and we don't want to imply value distinction --> onehotencoder to create dummies."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now we are going to split the dataset into training and test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2,\n",
    "                                                   random_state = 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "^^ 20% of the data will be used for testing, thus it is obvious that the training set is 0.8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now we will learn about feature scaling."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Age and salary have vastly different values, and ML algortihms often use euclidean distances. Thus the square of a salary value will vastly outweight the effect of age^2 - thus, we scale our values to fix this issue.\n",
    "\n",
    "There are two types of feature scaling:\n",
    "\n",
    "- Standardisation is:\n",
    "    (x-mean(x))/sd(x)\n",
    "\n",
    "- Normalization is:\n",
    "    (x-min(x))/(max(x)-min(x))\n",
    "\n",
    "Even in instances where ML algos dont use euclidean distances, we still need to feature scale as it will allow the ML algo to converge faster. This is especially true for decision trees ( we will come back to this way later)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sc_X = StandardScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X_train = sc_X.fit_transform(X_train)\n",
    "#note that we need to fit AND transform the training set\n",
    "#we will only transform the test set.\n",
    "X_test = sc_X.transform(X_test)\n",
    "#there is debate over whether we should scale the dummy variables.\n",
    "#it depends on the context and for interpretation\n",
    "#if we scale the dummy, then we will lose the interpretation of a 1\n",
    "#referring to a particular country, this makes interpretation weird\n",
    "#however, it could be beneficial for our model.\n",
    "#since this we are not going to interpret\n",
    "#we will scale everything, including the dummies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.        ,  2.64575131, -0.77459667,  0.26306757,  0.12381479],\n",
       "       [ 1.        , -0.37796447, -0.77459667, -0.25350148,  0.46175632],\n",
       "       [-1.        , -0.37796447,  1.29099445, -1.97539832, -1.53093341],\n",
       "       [-1.        , -0.37796447,  1.29099445,  0.05261351, -1.11141978],\n",
       "       [ 1.        , -0.37796447, -0.77459667,  1.64058505,  1.7202972 ],\n",
       "       [-1.        , -0.37796447,  1.29099445, -0.0813118 , -0.16751412],\n",
       "       [ 1.        , -0.37796447, -0.77459667,  0.95182631,  0.98614835],\n",
       "       [ 1.        , -0.37796447, -0.77459667, -0.59788085, -0.48214934]])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.        ,  2.64575131, -0.77459667, -1.45882927, -0.90166297],\n",
       "       [-1.        ,  2.64575131, -0.77459667,  1.98496442,  2.13981082]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Do we do feature scaling to the dependent value? No, in this instance we do not, becacuse we have a classification problem with a categorical dependent variable. However, later we will see in some regression problems where the dependent variable can take a large range of values - then we will deal with feature scaling."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "A confusion that might arise is, why do we type the follwing code:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# X_train = sc_X.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Should we not fit and transform the test set too? The answer is NO!\n",
    "\n",
    "the .fit_transform(X_train) essentially finds the 'fit' of the training set, that is, outliers, min, max etc, and then based on that fit, transforms the data (feature scaling). the fit is 'stored' in our Standard Scaler object and thus we neednt fit AND transform out X_test, rather, we need to just transform it based on the fit.\n",
    "\n",
    "An alternative way to write that code segement would be the following:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# sc_X = StandardScaler()\n",
    "# sc_X.fit(X_train)\n",
    "# X_train = sc_X.transform(X_train)\n",
    "# X_test = sc_X.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Another concern might be, why dont we have the fit on X rather than on X_train? THis is a valid point, however, since X_train represents 80% of X, we are taking a slight shortcut by fitting and transforming on X_train directly as it is obvious that X_train and X will have extrememly similar characteristics.\n",
    "\n",
    "This may bother some people, thus an alternative way to write this would be as follows:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# sc_X = StandardScaler()\n",
    "# sc_X.fit(X)\n",
    "# X_train = sc_X.transform(X_train)\n",
    "# X_test = sc_X.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}