W02_Prakash Panicker_qq328457 (1).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Your name:\n",
    "\n",
    "<pre> Nitin Prakash Panicker</pre>\n",
    "\n",
    "### Collaborators:\n",
    "\n",
    "<pre> </pre>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Open the housing data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>longitude</th>\n",
       "      <th>latitude</th>\n",
       "      <th>housing_median_age</th>\n",
       "      <th>total_rooms</th>\n",
       "      <th>total_bedrooms</th>\n",
       "      <th>population</th>\n",
       "      <th>households</th>\n",
       "      <th>median_income</th>\n",
       "      <th>median_house_value</th>\n",
       "      <th>ocean_proximity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-122.23</td>\n",
       "      <td>37.88</td>\n",
       "      <td>41.0</td>\n",
       "      <td>880.0</td>\n",
       "      <td>129.0</td>\n",
       "      <td>322.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>8.3252</td>\n",
       "      <td>452600.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-122.22</td>\n",
       "      <td>37.86</td>\n",
       "      <td>21.0</td>\n",
       "      <td>7099.0</td>\n",
       "      <td>1106.0</td>\n",
       "      <td>2401.0</td>\n",
       "      <td>1138.0</td>\n",
       "      <td>8.3014</td>\n",
       "      <td>358500.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-122.24</td>\n",
       "      <td>37.85</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1467.0</td>\n",
       "      <td>190.0</td>\n",
       "      <td>496.0</td>\n",
       "      <td>177.0</td>\n",
       "      <td>7.2574</td>\n",
       "      <td>352100.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-122.25</td>\n",
       "      <td>37.85</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1274.0</td>\n",
       "      <td>235.0</td>\n",
       "      <td>558.0</td>\n",
       "      <td>219.0</td>\n",
       "      <td>5.6431</td>\n",
       "      <td>341300.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-122.25</td>\n",
       "      <td>37.85</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1627.0</td>\n",
       "      <td>280.0</td>\n",
       "      <td>565.0</td>\n",
       "      <td>259.0</td>\n",
       "      <td>3.8462</td>\n",
       "      <td>342200.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   longitude  latitude  housing_median_age  total_rooms  total_bedrooms  \\\n",
       "0    -122.23     37.88                41.0        880.0           129.0   \n",
       "1    -122.22     37.86                21.0       7099.0          1106.0   \n",
       "2    -122.24     37.85                52.0       1467.0           190.0   \n",
       "3    -122.25     37.85                52.0       1274.0           235.0   \n",
       "4    -122.25     37.85                52.0       1627.0           280.0   \n",
       "\n",
       "   population  households  median_income  median_house_value ocean_proximity  \n",
       "0       322.0       126.0         8.3252            452600.0        NEAR BAY  \n",
       "1      2401.0      1138.0         8.3014            358500.0        NEAR BAY  \n",
       "2       496.0       177.0         7.2574            352100.0        NEAR BAY  \n",
       "3       558.0       219.0         5.6431            341300.0        NEAR BAY  \n",
       "4       565.0       259.0         3.8462            342200.0        NEAR BAY  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import tarfile\n",
    "from six.moves import urllib\n",
    "\n",
    "DOWNLOAD_ROOT = \"https://raw.githubusercontent.com/ageron/handson-ml/master/\"\n",
    "HOUSING_PATH = os.path.join(\"datasets\", \"housing\")\n",
    "HOUSING_URL = DOWNLOAD_ROOT + \"datasets/housing/housing.tgz\"\n",
    "\n",
    "def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):\n",
    "    if not os.path.isdir(housing_path):\n",
    "        os.makedirs(housing_path)\n",
    "    tgz_path = os.path.join(housing_path, \"housing.tgz\")\n",
    "    urllib.request.urlretrieve(housing_url, tgz_path)\n",
    "    housing_tgz = tarfile.open(tgz_path)\n",
    "    housing_tgz.extractall(path=housing_path)\n",
    "    housing_tgz.close()\n",
    "\n",
    "fetch_housing_data()\n",
    "import pandas as pd\n",
    "\n",
    "def load_housing_data(housing_path=HOUSING_PATH):\n",
    "    csv_path = os.path.join(housing_path, \"housing.csv\")\n",
    "    return pd.read_csv(csv_path)\n",
    "\n",
    "housing = load_housing_data()\n",
    "housing.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build a full pipeline for data analysis following the example of the module notebook.\n",
    " Hint: the main change is the algorithm used (Lasso regression)\n",
    "\n",
    "If you want to learn more about the Lasso regression, see resources below:\n",
    "- http://scikit-learn.org/stable/modules/linear_model.html#lasso\n",
    "- https://www.analyticsvidhya.com/blog/2016/01/complete-tutorial-ridge-lasso-regression-python/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Considerations for building a pipeline:\n",
    "\n",
    "- Split data into training and testing sets (1 mark)\n",
    "- Convert all categorical data to one-hot vectors below inside a pipeline (2 marks)\n",
    "- Normalize all non-categorical data inside the same pipeline (2 marks)\n",
    "- Perform Lasso-based regression (1 mark)\n",
    "- Perform a grid search using a variety of values for $\\alpha$ between 0 and 1 where *housing_labels* is the output and all other features are the input (2 marks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 20640 entries, 0 to 20639\n",
      "Data columns (total 10 columns):\n",
      "longitude             20640 non-null float64\n",
      "latitude              20640 non-null float64\n",
      "housing_median_age    20640 non-null float64\n",
      "total_rooms           20640 non-null float64\n",
      "total_bedrooms        20433 non-null float64\n",
      "population            20640 non-null float64\n",
      "households            20640 non-null float64\n",
      "median_income         20640 non-null float64\n",
      "median_house_value    20640 non-null float64\n",
      "ocean_proximity       20640 non-null object\n",
      "dtypes: float64(9), object(1)\n",
      "memory usage: 1.6+ MB\n"
     ]
    }
   ],
   "source": [
    "housing.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAD8CAYAAACRkhiPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFt9JREFUeJzt3XGMHPdZxvHvg92mTlw3DmkW47OwARNwbJrWJ8sQFZ1xSgwJdf4g0lWhOUPQociUFhmRM0gg/jBYggCN2gRZTWtHCT1ZoVGsui41JiuE5Da107RXxzU+GuNcfNilNK0vrVIuvPyxv7iT897drm9vZ7e/5yOtduadmd13b+/u2fnN7K4iAjMzy9OPlN2AmZmVxyFgZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllbGHZDczm+uuvj5UrV16af+WVV7jmmmvKa6hB7rO13Gdruc/W6sQ+jx8//t8R8fZZV4yIjr6sX78+ip5++unoBu6ztdxna7nP1urEPoFj0cD/WA8HmZllzCFgZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllrOM/NqIbrRw6yI51k2wbOtj2+z6z+/a236eZdS/vCZiZZcwhYGaWMYeAmVnGHAJmZhlzCJiZZcwhYGaWMYeAmVnGZg0BSTdKeq5w+Y6kD0m6TtJhSafT9dLCNjsljUo6Jem2Qn29pJG07EFJmq8HZmZms5s1BCLiVETcHBE3A+uB7wJPAkPAkYhYDRxJ80haA/QDNwFbgIckLUg39zAwCKxOly2tfThmZtaMZoeDNgP/ERH/CWwF9qX6PuDONL0VGI6IVyPiBWAU2CBpGbAkIo6m7798tLCNmZmVoNkQ6Ac+maYrETEOkK5vSPXlwIuFbcZSbXmanlo3M7OSqPaivIEVpTcD54CbIuK8pJcj4trC8m9FxFJJHwWORsRjqf4I8BngLPCXEXFrqr8b+KOI+PU69zVIbdiISqWyfnh4+NKyiYkJFi9efGWPtk1GXvo2lUVw/nvtv+91y9/W1Prd8PME99lq7rO1OrHPTZs2HY+I3tnWa+YD5H4VeDYizqf585KWRcR4Guq5kOpjwIrCdj3UwmMsTU+tXyYi9gB7AHp7e6Ovr+/Ssmq1SnG+E21LHyD3wEj7P5/vzN19Ta3fDT9PcJ+t5j5bq1v6rKeZ4aD38YOhIIADwECaHgCeKtT7JV0laRW1A8DPpCGji5I2prOC7ilsY2ZmJWjopaqkq4H3AL9bKO8G9ku6l9pQz10AEXFC0n7geWAS2B4Rr6Vt7gP2AouAQ+liZmYlaSgEIuK7wI9OqX2T2tlC9dbfBeyqUz8GrG2+TTMzmw9+x7CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllrKEQkHStpCckfU3SSUm/IOk6SYclnU7XSwvr75Q0KumUpNsK9fWSRtKyByVpPh6UmZk1pqEvmgc+DHw2In5D0puBq4E/Bo5ExG5JQ8AQcL+kNUA/cBPw48A/S/qZiHgNeBgYBD4PfAbYAhxq6SPK3Mqhg02tv2PdJNua3KaeM7tvn/NtmFn7zbonIGkJ8EvAIwAR8f2IeBnYCuxLq+0D7kzTW4HhiHg1Il4ARoENkpYBSyLiaEQE8GhhGzMzK0Ejw0E/CXwD+ISkL0n6mKRrgEpEjAOk6xvS+suBFwvbj6Xa8jQ9tW5mZiVpZDhoIfAu4AMR8QVJH6Y29DOdeuP8MUP98huQBqkNG1GpVKhWq5eWTUxMvGG+E+1YN0llUe2607Wqz/l+TrrheQf32Wruc/41EgJjwFhEfCHNP0EtBM5LWhYR42mo50Jh/RWF7XuAc6neU6d+mYjYA+wB6O3tjb6+vkvLqtUqxflOtG3oIDvWTfLASKOHXMrTqj7P3N0392Zm0A3PO7jPVnOf82/W4aCI+C/gRUk3ptJm4HngADCQagPAU2n6ANAv6SpJq4DVwDNpyOiipI3prKB7CtuYmVkJGn0J+AHg8XRm0NeB36IWIPsl3QucBe4CiIgTkvZTC4pJYHs6MwjgPmAvsIjaWUE+M8jMrEQNhUBEPAf01lm0eZr1dwG76tSPAWubadDMzOaP3zFsZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGWsoBCSdkTQi6TlJx1LtOkmHJZ1O10sL6++UNCrplKTbCvX16XZGJT0oSa1/SGZm1qhm9gQ2RcTNEfH6F84PAUciYjVwJM0jaQ3QD9wEbAEekrQgbfMwMAisTpctc38IZmZ2peYyHLQV2Jem9wF3FurDEfFqRLwAjAIbJC0DlkTE0YgI4NHCNmZmVoJGQyCAz0k6Lmkw1SoRMQ6Qrm9I9eXAi4Vtx1JteZqeWjczs5IsbHC9WyLinKQbgMOSvjbDuvXG+WOG+uU3UAuaQYBKpUK1Wr20bGJi4g3znWjHukkqi2rXna5Vfc73c9INzzu4z1Zzn/OvoRCIiHPp+oKkJ4ENwHlJyyJiPA31XEirjwErCpv3AOdSvadOvd797QH2APT29kZfX9+lZdVqleJ8J9o2dJAd6yZ5YKTRjC1Pq/o8c3ff3JuZQTc87+A+W819zr9Zh4MkXSPpra9PA78CfBU4AAyk1QaAp9L0AaBf0lWSVlE7APxMGjK6KGljOivonsI2ZmZWgkZeAlaAJ9PZnAuBf4iIz0r6IrBf0r3AWeAugIg4IWk/8DwwCWyPiNfSbd0H7AUWAYfSxczMSjJrCETE14F31Kl/E9g8zTa7gF116seAtc23aWZm88HvGDYzy5hDwMwsYw4BM7OMOQTMzDLmEDAzy5hDwMwsYw4BM7OMOQTMzDLmEDAzy5hDwMwsYw4BM7OMOQTMzDLmEDAzy5hDwMwsYw4BM7OMOQTMzDLmEDAzy5hDwMwsYw4BM7OMNRwCkhZI+pKkT6f56yQdlnQ6XS8trLtT0qikU5JuK9TXSxpJyx5U+vZ6MzMrRzN7Ah8EThbmh4AjEbEaOJLmkbQG6AduArYAD0lakLZ5GBgEVqfLljl1b2Zmc9JQCEjqAW4HPlYobwX2pel9wJ2F+nBEvBoRLwCjwAZJy4AlEXE0IgJ4tLCNmZmVQLX/x7OsJD0B/CXwVuAPI+IOSS9HxLWFdb4VEUslfQT4fEQ8luqPAIeAM8DuiLg11d8N3B8Rd9S5v0FqewxUKpX1w8PDl5ZNTEywePHiK328bTHy0repLILz3yu7k9m1qs91y9829xuZQTc87+A+W819XrlNmzYdj4je2dZbONsKku4ALkTEcUl9Ddx3vXH+mKF+eTFiD7AHoLe3N/r6fnC31WqV4nwn2jZ0kB3rJnlgZNYfb+la1eeZu/vm3swMuuF5B/fZau5z/jXy138L8F5Jvwa8BVgi6THgvKRlETGehnoupPXHgBWF7XuAc6neU6duZmYlmfWYQETsjIieiFhJ7YDvv0TEbwIHgIG02gDwVJo+APRLukrSKmoHgJ+JiHHgoqSN6aygewrbmJlZCeYyDrAb2C/pXuAscBdARJyQtB94HpgEtkfEa2mb+4C9wCJqxwkOzeH+zcxsjpoKgYioAtU0/U1g8zTr7QJ21akfA9Y226SZmc0Pv2PYzCxjDgEzs4w5BMzMMuYQMDPLmEPAzCxjDgEzs4w5BMzMMuYQMDPLmEPAzCxjDgEzs4w5BMzMMuYQMDPLmEPAzCxjDgEzs4w5BMzMMuYQMDPLmEPAzCxjDgEzs4w5BMzMMjZrCEh6i6RnJH1Z0glJf57q10k6LOl0ul5a2GanpFFJpyTdVqivlzSSlj0oSfPzsMzMrBGN7Am8CvxyRLwDuBnYImkjMAQciYjVwJE0j6Q1QD9wE7AFeEjSgnRbDwODwOp02dLCx2JmZk2aNQSiZiLNvildAtgK7Ev1fcCdaXorMBwRr0bEC8AosEHSMmBJRByNiAAeLWxjZmYlUO3/8Swr1V7JHwd+GvhoRNwv6eWIuLawzrciYqmkjwCfj4jHUv0R4BBwBtgdEbem+ruB+yPijjr3N0htj4FKpbJ+eHj40rKJiQkWL158pY+3LUZe+jaVRXD+e2V3MrtW9blu+dvmfiMz6IbnHdxnq7nPK7dp06bjEdE723oLG7mxiHgNuFnStcCTktbOsHq9cf6YoV7v/vYAewB6e3ujr6/v0rJqtUpxvhNtGzrIjnWTPDDS0I+3VC3rc+SVud/GDHase40H/q3+fZzZffu83nczuuH3E9xnq3VLn/U0dXZQRLwMVKmN5Z9PQzyk6wtptTFgRWGzHuBcqvfUqZuZWUkaOTvo7WkPAEmLgFuBrwEHgIG02gDwVJo+APRLukrSKmoHgJ+JiHHgoqSN6aygewrbmJlZCRoZB1gG7EvHBX4E2B8Rn5Z0FNgv6V7gLHAXQESckLQfeB6YBLan4SSA+4C9wCJqxwkOtfLBmJlZc2YNgYj4CvDOOvVvApun2WYXsKtO/Rgw0/EEMzNrI79j2MwsYw4BM7OMOQTMzDLmEDAzy5hDwMwsYw4BM7OMOQTMzDLmEDAzy5hDwMwsYw4BM7OMOQTMzDLmEDAzy5hDwMwsYw4BM7OMOQTMzDLW+V+COwcrhw6W3YKZWUfznoCZWcYcAmZmGWvki+ZXSHpa0klJJyR9MNWvk3RY0ul0vbSwzU5Jo5JOSbqtUF8vaSQtezB94byZmZWkkT2BSWBHRPwcsBHYLmkNMAQciYjVwJE0T1rWD9wEbAEeSl9SD/AwMAisTpctLXwsZmbWpFlDICLGI+LZNH0ROAksB7YC+9Jq+4A70/RWYDgiXo2IF4BRYIOkZcCSiDgaEQE8WtjGzMxK0NQxAUkrgXcCXwAqETEOtaAAbkirLQdeLGw2lmrL0/TUupmZlaThU0QlLQb+EfhQRHxnhuH8egtihnq9+xqkNmxEpVKhWq1eWjYxMfGG+ZnsWDfZ0HrzobKo3Ptv1A9Dn43+PrRDM7+fZXKfrdUtfdbTUAhIehO1AHg8Ij6VyuclLYuI8TTUcyHVx4AVhc17gHOp3lOnfpmI2APsAejt7Y2+vr5Ly6rVKsX5mWwr8X0CO9ZN8sBI578N44ehzzN397W3mRk08/tZJvfZWt3SZz2NnB0k4BHgZET8TWHRAWAgTQ8ATxXq/ZKukrSK2gHgZ9KQ0UVJG9Nt3lPYxszMStDIS8BbgPcDI5KeS7U/BnYD+yXdC5wF7gKIiBOS9gPPUzuzaHtEvJa2uw/YCywCDqWLmZmVZNYQiIh/o/54PsDmabbZBeyqUz8GrG2mQTMzmz9+x7CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllbNYQkPRxSRckfbVQu07SYUmn0/XSwrKdkkYlnZJ0W6G+XtJIWvagpOm+t9jMzNqkkT2BvcCWKbUh4EhErAaOpHkkrQH6gZvSNg9JWpC2eRgYBFany9TbNDOzNls42woR8a+SVk4pbwX60vQ+oArcn+rDEfEq8IKkUWCDpDPAkog4CiDpUeBO4NCcH4Flb+XQwVLu98zu20u5X7NWutJjApWIGAdI1zek+nLgxcJ6Y6m2PE1PrZuZWYlm3RNoUr1x/pihXv9GpEFqQ0dUKhWq1eqlZRMTE2+Yn8mOdZMNrTcfKovKvf9Guc8rV+/3sJnfzzK5z9bqlj7rudIQOC9pWUSMS1oGXEj1MWBFYb0e4Fyq99Sp1xURe4A9AL29vdHX13dpWbVapTg/k20lDRNA7R/WAyOtztjWc59X7szdfZfVmvn9LJP7bK1u6bOeKx0OOgAMpOkB4KlCvV/SVZJWUTsA/EwaMrooaWM6K+iewjZmZlaSWV9aSfoktYPA10saA/4M2A3sl3QvcBa4CyAiTkjaDzwPTALbI+K1dFP3UTvTaBG1A8I+KGxmVrJGzg563zSLNk+z/i5gV536MWBtU92Zmdm88juGzcwy5hAwM8uYQ8DMLGMOATOzjDkEzMwy5hAwM8uYQ8DMLGMOATOzjDkEzMwy5hAwM8uYQ8DMLGOd9dm8Zl2k3jea7Vg32ZaPMPe3mlmreE/AzCxjDgEzs4w5BMzMMuYQMDPLmEPAzCxjDgEzs4z5FFGzLlTv9NRmXOmprD419YdP2/cEJG2RdErSqKShdt+/mZn9QFv3BCQtAD4KvAcYA74o6UBEPN/OPszsysx1D6RZr++xeA9k/rR7OGgDMBoRXweQNAxsBRwCZjatdodPs+bjneLtCr52DwctB14szI+lmpmZlUAR0b47k+4CbouI30nz7wc2RMQHpqw3CAym2RuBU4XF1wP/3YZ258p9tpb7bC332Vqd2OdPRMTbZ1up3cNBY8CKwnwPcG7qShGxB9hT7wYkHYuI3vlpr3XcZ2u5z9Zyn63VLX3W0+7hoC8CqyWtkvRmoB840OYezMwsaeueQERMSvo94J+ABcDHI+JEO3swM7MfaPubxSLiM8Bn5nATdYeJOpD7bC332Vrus7W6pc/LtPXAsJmZdRZ/dpCZWca6KgS64SMnJK2Q9LSkk5JOSPpg2T3NRNICSV+S9Omye5mOpGslPSHpa+nn+gtl91SPpD9Iz/lXJX1S0lvK7glA0sclXZD01ULtOkmHJZ1O10vL7DH1VK/Pv0rP+1ckPSnp2jJ7TD1d1mdh2R9KCknXl9HbleiaECh85MSvAmuA90laU25XdU0COyLi54CNwPYO7fN1HwROlt3ELD4MfDYifhZ4Bx3Yr6TlwO8DvRGxltqJD/3ldnXJXmDLlNoQcCQiVgNH0nzZ9nJ5n4eBtRHx88C/Azvb3VQde7m8TyStoPaROGfb3dBcdE0IUPjIiYj4PvD6R050lIgYj4hn0/RFav+wOvJd0ZJ6gNuBj5Xdy3QkLQF+CXgEICK+HxEvl9vVtBYCiyQtBK6mzntgyhAR/wr8z5TyVmBfmt4H3NnWpuqo12dEfC4iJtPs56m9t6hU0/w8Af4W+COgqw60dlMIdN1HTkhaCbwT+EK5nUzr76j90v5f2Y3M4CeBbwCfSMNWH5N0TdlNTRURLwF/Te1V4Djw7Yj4XLldzagSEeNQe+EC3FByP434beBQ2U3UI+m9wEsR8eWye2lWN4WA6tQ6NnElLQb+EfhQRHyn7H6mknQHcCEijpfdyywWAu8CHo6IdwKv0BlDF2+QxtS3AquAHweukfSb5Xb1w0PSn1Aban287F6mknQ18CfAn5bdy5XophBo6CMnOoGkN1ELgMcj4lNl9zONW4D3SjpDbWjtlyU9Vm5LdY0BYxHx+t7UE9RCodPcCrwQEd+IiP8FPgX8Ysk9zeS8pGUA6fpCyf1MS9IAcAdwd3TmOe0/RS38v5z+nnqAZyX9WKldNaibQqArPnJCkqiNX5+MiL8pu5/pRMTOiOiJiJXUfpb/EhEd98o1Iv4LeFHSjam0mc786PGzwEZJV6ffgc104AHsggPAQJoeAJ4qsZdpSdoC3A+8NyK+W3Y/9UTESETcEBEr09/TGPCu9Lvb8bomBNLBodc/cuIksL9DP3LiFuD91F5ZP5cuv1Z2U13uA8Djkr4C3Az8Rcn9XCbtqTwBPAuMUPvb6oh3kUr6JHAUuFHSmKR7gd3AeySdpnZGy+4ye4Rp+/wI8FbgcPpb+vtSm2TaPruW3zFsZpaxrtkTMDOz1nMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcb+H93AdE6EEamhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# As per experts (business), median income is a very important attribute to predict housing prices. \n",
    "# Since median income is a continuous numerical attribute, we need to first create an income category attribute. \n",
    "\n",
    "#  Let’s look at the median income histogram more closely\n",
    "housing[\"median_income\"].hist();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The following code creates an income category \n",
    "# attribute by dividing the median income by 1.5 (to limit the number of income cate‐gories), \n",
    "# and rounding up using ceil (to have discrete categories), and then merging\n",
    "# all the categories greater than 5 into category 5\n",
    "\n",
    "housing[\"income_cat\"] = np.ceil(housing[\"median_income\"] / 1.5)\n",
    "housing[\"income_cat\"].where(housing[\"income_cat\"] < 5, 5.0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.0    7236\n",
       "2.0    6581\n",
       "4.0    3639\n",
       "5.0    2362\n",
       "1.0     822\n",
       "Name: income_cat, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing[\"income_cat\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train-Test split based on stratified sampling technique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Now we are ready to do stratified sampling based on the income category. \n",
    "# For this we can use Scikit-Learn’s StratifiedShuffleSplit class\n",
    "\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "\n",
    "split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n",
    "for train_index, test_index in split.split(housing, housing[\"income_cat\"]):\n",
    "    strat_train_set = housing.loc[train_index]\n",
    "    strat_test_set = housing.loc[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.0    0.350581\n",
       "2.0    0.318847\n",
       "4.0    0.176308\n",
       "5.0    0.114438\n",
       "1.0    0.039826\n",
       "Name: income_cat, dtype: float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# income category proportions in the full housing dataset\n",
    "\n",
    "housing[\"income_cat\"].value_counts() / len(housing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Now you should remove the income_cat attribute so the data is back to its original state:\n",
    "\n",
    "for set_ in (strat_train_set, strat_test_set):\n",
    "    set_.drop(\"income_cat\", axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "housing = strat_train_set.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attribute combinations:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# the total number of rooms in a district is not very useful if you don’t know how many\n",
    "# households there are. What you really want is the number of rooms per household.\n",
    "# Similarly, the total number of bedrooms by itself is not very useful: you probably\n",
    "# want to compare it to the number of rooms. And the population per household also\n",
    "# seems like an interesting attribute combination to look at. Let’s create these new\n",
    "# attributes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "housing[\"rooms_per_household\"] = housing[\"total_rooms\"]/housing[\"households\"]\n",
    "housing[\"bedrooms_per_room\"] = housing[\"total_bedrooms\"]/housing[\"total_rooms\"]\n",
    "housing[\"population_per_household\"]=housing[\"population\"]/housing[\"households\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "median_house_value          1.000000\n",
       "median_income               0.687160\n",
       "rooms_per_household         0.146285\n",
       "total_rooms                 0.135097\n",
       "housing_median_age          0.114110\n",
       "households                  0.064506\n",
       "total_bedrooms              0.047689\n",
       "population_per_household   -0.021985\n",
       "population                 -0.026920\n",
       "longitude                  -0.047432\n",
       "latitude                   -0.142724\n",
       "bedrooms_per_room          -0.259984\n",
       "Name: median_house_value, dtype: float64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# And now let’s look at the correlation matrix again:\n",
    "corr_matrix = housing.corr()\n",
    "corr_matrix[\"median_house_value\"].sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare data for machine learning algorithms:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# let’s revert to a clean training set (by copying strat_train_set once again),\n",
    "# and let’s separate the predictors and the labels since we don’t necessarily want to apply\n",
    "# the same transformations to the predictors and the target values\n",
    "\n",
    "housing = strat_train_set.drop(\"median_house_value\", axis=1)\n",
    "housing_labels = strat_train_set[\"median_house_value\"].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Fill missing values for total_bedrooms using sci-kit learn Imputer\n",
    "try:\n",
    "    from sklearn.impute import SimpleImputer # Scikit-Learn 0.20+\n",
    "except ImportError:\n",
    "    from sklearn.preprocessing import Imputer as SimpleImputer\n",
    "\n",
    "imputer = SimpleImputer(strategy=\"median\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Since the median can only be computed on numerical attributes, we need to create a\n",
    "# copy of the data without the text attribute ocean_proximity:\n",
    "\n",
    "housing_num = housing.drop(\"ocean_proximity\", axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SimpleImputer(copy=True, fill_value=None, missing_values=nan,\n",
       "       strategy='median', verbose=0)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now you can fit the imputer instance to the training data using the fit() method:\n",
    "imputer.fit(housing_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Transfer the training set:\n",
    "# Now we can use this “trained” imputer to transform the training set by replacing missing values by the learned medians:\n",
    "\n",
    "X = imputer.transform(housing_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The result is a plain Numpy array containing the transformed features. If you want to\n",
    "# put it back into a Pandas DataFrame, it’s simple:\n",
    "\n",
    "housing_tr = pd.DataFrame(X, columns=housing_num.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>longitude</th>\n",
       "      <th>latitude</th>\n",
       "      <th>housing_median_age</th>\n",
       "      <th>total_rooms</th>\n",
       "      <th>total_bedrooms</th>\n",
       "      <th>population</th>\n",
       "      <th>households</th>\n",
       "      <th>median_income</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-121.89</td>\n",
       "      <td>37.29</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1568.0</td>\n",
       "      <td>351.0</td>\n",
       "      <td>710.0</td>\n",
       "      <td>339.0</td>\n",
       "      <td>2.7042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-121.93</td>\n",
       "      <td>37.05</td>\n",
       "      <td>14.0</td>\n",
       "      <td>679.0</td>\n",
       "      <td>108.0</td>\n",
       "      <td>306.0</td>\n",
       "      <td>113.0</td>\n",
       "      <td>6.4214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-117.20</td>\n",
       "      <td>32.77</td>\n",
       "      <td>31.0</td>\n",
       "      <td>1952.0</td>\n",
       "      <td>471.0</td>\n",
       "      <td>936.0</td>\n",
       "      <td>462.0</td>\n",
       "      <td>2.8621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-119.61</td>\n",
       "      <td>36.31</td>\n",
       "      <td>25.0</td>\n",
       "      <td>1847.0</td>\n",
       "      <td>371.0</td>\n",
       "      <td>1460.0</td>\n",
       "      <td>353.0</td>\n",
       "      <td>1.8839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-118.59</td>\n",
       "      <td>34.23</td>\n",
       "      <td>17.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>1525.0</td>\n",
       "      <td>4459.0</td>\n",
       "      <td>1463.0</td>\n",
       "      <td>3.0347</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   longitude  latitude  housing_median_age  total_rooms  total_bedrooms  \\\n",
       "0    -121.89     37.29                38.0       1568.0           351.0   \n",
       "1    -121.93     37.05                14.0        679.0           108.0   \n",
       "2    -117.20     32.77                31.0       1952.0           471.0   \n",
       "3    -119.61     36.31                25.0       1847.0           371.0   \n",
       "4    -118.59     34.23                17.0       6592.0          1525.0   \n",
       "\n",
       "   population  households  median_income  \n",
       "0       710.0       339.0         2.7042  \n",
       "1       306.0       113.0         6.4214  \n",
       "2       936.0       462.0         2.8621  \n",
       "3      1460.0       353.0         1.8839  \n",
       "4      4459.0      1463.0         3.0347  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing_tr.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Handling Text and Categorical Attributes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ocean_proximity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17606</th>\n",
       "      <td>&lt;1H OCEAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18632</th>\n",
       "      <td>&lt;1H OCEAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14650</th>\n",
       "      <td>NEAR OCEAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3230</th>\n",
       "      <td>INLAND</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3555</th>\n",
       "      <td>&lt;1H OCEAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480</th>\n",
       "      <td>INLAND</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8879</th>\n",
       "      <td>&lt;1H OCEAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13685</th>\n",
       "      <td>INLAND</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4937</th>\n",
       "      <td>&lt;1H OCEAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4861</th>\n",
       "      <td>&lt;1H OCEAN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ocean_proximity\n",
       "17606       <1H OCEAN\n",
       "18632       <1H OCEAN\n",
       "14650      NEAR OCEAN\n",
       "3230           INLAND\n",
       "3555        <1H OCEAN\n",
       "19480          INLAND\n",
       "8879        <1H OCEAN\n",
       "13685          INLAND\n",
       "4937        <1H OCEAN\n",
       "4861        <1H OCEAN"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing_cat = housing[['ocean_proximity']]\n",
    "housing_cat.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OrdinalEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.],\n",
       "       [0.],\n",
       "       [4.],\n",
       "       [1.],\n",
       "       [0.],\n",
       "       [1.],\n",
       "       [0.],\n",
       "       [1.],\n",
       "       [0.],\n",
       "       [0.]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ordinal_encoder = OrdinalEncoder()\n",
    "housing_cat_encoded = ordinal_encoder.fit_transform(housing_cat)\n",
    "housing_cat_encoded[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array(['<1H OCEAN', 'INLAND', 'ISLAND', 'NEAR BAY', 'NEAR OCEAN'],\n",
       "       dtype=object)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ordinal_encoder.categories_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0., 0., 0., 0.],\n",
       "       [1., 0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 1.],\n",
       "       ...,\n",
       "       [0., 1., 0., 0., 0.],\n",
       "       [1., 0., 0., 0., 0.],\n",
       "       [0., 0., 0., 1., 0.]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_encoder = OneHotEncoder(sparse=False)\n",
    "housing_cat_1hot = cat_encoder.fit_transform(housing_cat)\n",
    "housing_cat_1hot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Custom Transformers to  add extra attributes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "\n",
    "# get the right column indices: safer than hard-coding indices 3, 4, 5, 6\n",
    "rooms_ix, bedrooms_ix, population_ix, household_ix = [\n",
    "    list(housing.columns).index(col)\n",
    "    for col in (\"total_rooms\", \"total_bedrooms\", \"population\", \"households\")]\n",
    "\n",
    "def add_extra_features(X, add_bedrooms_per_room=True):\n",
    "    rooms_per_household = X[:, rooms_ix] / X[:, household_ix]\n",
    "    population_per_household = X[:, population_ix] / X[:, household_ix]\n",
    "    if add_bedrooms_per_room:\n",
    "        bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]\n",
    "        return np.c_[X, rooms_per_household, population_per_household,\n",
    "                     bedrooms_per_room]\n",
    "    else:\n",
    "        return np.c_[X, rooms_per_household, population_per_household]\n",
    "\n",
    "attr_adder = FunctionTransformer(add_extra_features, validate=False,\n",
    "                                 kw_args={\"add_bedrooms_per_room\": False})\n",
    "housing_extra_attribs = attr_adder.fit_transform(housing.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>longitude</th>\n",
       "      <th>latitude</th>\n",
       "      <th>housing_median_age</th>\n",
       "      <th>total_rooms</th>\n",
       "      <th>total_bedrooms</th>\n",
       "      <th>population</th>\n",
       "      <th>households</th>\n",
       "      <th>median_income</th>\n",
       "      <th>ocean_proximity</th>\n",
       "      <th>rooms_per_household</th>\n",
       "      <th>population_per_household</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-121.89</td>\n",
       "      <td>37.29</td>\n",
       "      <td>38</td>\n",
       "      <td>1568</td>\n",
       "      <td>351</td>\n",
       "      <td>710</td>\n",
       "      <td>339</td>\n",
       "      <td>2.7042</td>\n",
       "      <td>&lt;1H OCEAN</td>\n",
       "      <td>4.62537</td>\n",
       "      <td>2.0944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-121.93</td>\n",
       "      <td>37.05</td>\n",
       "      <td>14</td>\n",
       "      <td>679</td>\n",
       "      <td>108</td>\n",
       "      <td>306</td>\n",
       "      <td>113</td>\n",
       "      <td>6.4214</td>\n",
       "      <td>&lt;1H OCEAN</td>\n",
       "      <td>6.00885</td>\n",
       "      <td>2.70796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-117.2</td>\n",
       "      <td>32.77</td>\n",
       "      <td>31</td>\n",
       "      <td>1952</td>\n",
       "      <td>471</td>\n",
       "      <td>936</td>\n",
       "      <td>462</td>\n",
       "      <td>2.8621</td>\n",
       "      <td>NEAR OCEAN</td>\n",
       "      <td>4.22511</td>\n",
       "      <td>2.02597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-119.61</td>\n",
       "      <td>36.31</td>\n",
       "      <td>25</td>\n",
       "      <td>1847</td>\n",
       "      <td>371</td>\n",
       "      <td>1460</td>\n",
       "      <td>353</td>\n",
       "      <td>1.8839</td>\n",
       "      <td>INLAND</td>\n",
       "      <td>5.23229</td>\n",
       "      <td>4.13598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-118.59</td>\n",
       "      <td>34.23</td>\n",
       "      <td>17</td>\n",
       "      <td>6592</td>\n",
       "      <td>1525</td>\n",
       "      <td>4459</td>\n",
       "      <td>1463</td>\n",
       "      <td>3.0347</td>\n",
       "      <td>&lt;1H OCEAN</td>\n",
       "      <td>4.50581</td>\n",
       "      <td>3.04785</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  longitude latitude housing_median_age total_rooms total_bedrooms population  \\\n",
       "0   -121.89    37.29                 38        1568            351        710   \n",
       "1   -121.93    37.05                 14         679            108        306   \n",
       "2    -117.2    32.77                 31        1952            471        936   \n",
       "3   -119.61    36.31                 25        1847            371       1460   \n",
       "4   -118.59    34.23                 17        6592           1525       4459   \n",
       "\n",
       "  households median_income ocean_proximity rooms_per_household  \\\n",
       "0        339        2.7042       <1H OCEAN             4.62537   \n",
       "1        113        6.4214       <1H OCEAN             6.00885   \n",
       "2        462        2.8621      NEAR OCEAN             4.22511   \n",
       "3        353        1.8839          INLAND             5.23229   \n",
       "4       1463        3.0347       <1H OCEAN             4.50581   \n",
       "\n",
       "  population_per_household  \n",
       "0                   2.0944  \n",
       "1                  2.70796  \n",
       "2                  2.02597  \n",
       "3                  4.13598  \n",
       "4                  3.04785  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing_extra_attribs = pd.DataFrame(\n",
    "    housing_extra_attribs,\n",
    "    columns=list(housing.columns)+[\"rooms_per_household\", \"population_per_household\"])\n",
    "housing_extra_attribs.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transformation Pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "num_pipeline = Pipeline([\n",
    "        ('imputer', SimpleImputer(strategy=\"median\")),\n",
    "        ('attribs_adder', FunctionTransformer(add_extra_features, validate=False)),\n",
    "        ('std_scaler', StandardScaler()),\n",
    "    ])\n",
    "\n",
    "housing_num_tr = num_pipeline.fit_transform(housing_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.15604281,  0.77194962,  0.74333089, ..., -0.31205452,\n",
       "        -0.08649871,  0.15531753],\n",
       "       [-1.17602483,  0.6596948 , -1.1653172 , ...,  0.21768338,\n",
       "        -0.03353391, -0.83628902],\n",
       "       [ 1.18684903, -1.34218285,  0.18664186, ..., -0.46531516,\n",
       "        -0.09240499,  0.4222004 ],\n",
       "       ...,\n",
       "       [ 1.58648943, -0.72478134, -1.56295222, ...,  0.3469342 ,\n",
       "        -0.03055414, -0.52177644],\n",
       "       [ 0.78221312, -0.85106801,  0.18664186, ...,  0.02499488,\n",
       "         0.06150916, -0.30340741],\n",
       "       [-1.43579109,  0.99645926,  1.85670895, ..., -0.22852947,\n",
       "        -0.09586294,  0.10180567]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing_num_tr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.compose import ColumnTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_attribs = list(housing_num)\n",
    "cat_attribs = [\"ocean_proximity\"]\n",
    "\n",
    "full_pipeline = ColumnTransformer([\n",
    "        (\"num\", num_pipeline, num_attribs),\n",
    "        (\"cat\", OneHotEncoder(), cat_attribs),\n",
    "    ])\n",
    "\n",
    "#housing_prepared = full_pipeline.fit_transform(housing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.15604281,  0.77194962,  0.74333089, ...,  0.        ,\n",
       "         0.        ,  0.        ],\n",
       "       [-1.17602483,  0.6596948 , -1.1653172 , ...,  0.        ,\n",
       "         0.        ,  0.        ],\n",
       "       [ 1.18684903, -1.34218285,  0.18664186, ...,  0.        ,\n",
       "         0.        ,  1.        ],\n",
       "       ...,\n",
       "       [ 1.58648943, -0.72478134, -1.56295222, ...,  0.        ,\n",
       "         0.        ,  0.        ],\n",
       "       [ 0.78221312, -0.85106801,  0.18664186, ...,  0.        ,\n",
       "         0.        ,  0.        ],\n",
       "       [-1.43579109,  0.99645926,  1.85670895, ...,  0.        ,\n",
       "         1.        ,  0.        ]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing_prepared = full_pipeline.fit_transform(housing)\n",
    "housing_prepared"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16512, 16)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing_prepared.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/srv/conda/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.\n",
      "  ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Lasso(alpha=1.0, copy_X=True, fit_intercept=True, max_iter=1000,\n",
       "   normalize=False, positive=False, precompute=False, random_state=None,\n",
       "   selection='cyclic', tol=0.0001, warm_start=False)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import Lasso\n",
    "\n",
    "# Write your code here:\n",
    "lasso = Lasso()\n",
    "lasso.fit(housing_prepared, housing_labels)\n",
    "#lasso_error = calculate_rmse(lasso, 'Lasso Reg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predictions: [210648.52944726 317763.26912906 210947.87343326  59226.96076999\n",
      " 189738.74813089]\n"
     ]
    }
   ],
   "source": [
    "# let's try the full preprocessing pipeline on a few training instances\n",
    "some_data = housing.iloc[:5]\n",
    "some_labels = housing_labels.iloc[:5]\n",
    "some_data_prepared = full_pipeline.transform(some_data)\n",
    "\n",
    "print(\"Predictions:\", lasso.predict(some_data_prepared))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Labels: [286600.0, 340600.0, 196900.0, 46300.0, 254500.0]\n"
     ]
    }
   ],
   "source": [
    "print(\"Labels:\", list(some_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.15604281,  0.77194962,  0.74333089, -0.49323393, -0.44543821,\n",
       "        -0.63621141, -0.42069842, -0.61493744, -0.31205452, -0.08649871,\n",
       "         0.15531753,  1.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ],\n",
       "       [-1.17602483,  0.6596948 , -1.1653172 , -0.90896655, -1.0369278 ,\n",
       "        -0.99833135, -1.02222705,  1.33645936,  0.21768338, -0.03353391,\n",
       "        -0.83628902,  1.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ],\n",
       "       [ 1.18684903, -1.34218285,  0.18664186, -0.31365989, -0.15334458,\n",
       "        -0.43363936, -0.0933178 , -0.5320456 , -0.46531516, -0.09240499,\n",
       "         0.4222004 ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         1.        ],\n",
       "       [-0.01706767,  0.31357576, -0.29052016, -0.36276217, -0.39675594,\n",
       "         0.03604096, -0.38343559, -1.04556555, -0.07966124,  0.08973561,\n",
       "        -0.19645314,  0.        ,  1.        ,  0.        ,  0.        ,\n",
       "         0.        ],\n",
       "       [ 0.49247384, -0.65929936, -0.92673619,  1.85619316,  2.41221109,\n",
       "         2.72415407,  2.57097492, -0.44143679, -0.35783383, -0.00419445,\n",
       "         0.2699277 ,  1.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ]])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "some_data_prepared"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "68628.2590530929"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "housing_predictions = lasso.predict(housing_prepared)\n",
    "lasso_mse = mean_squared_error(housing_labels, housing_predictions)\n",
    "lasso_rmse = np.sqrt(lasso_mse)\n",
    "lasso_rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "49440.168986993245"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import mean_absolute_error\n",
    "\n",
    "lasso_mae = mean_absolute_error(housing_labels, housing_predictions)\n",
    "lasso_mae"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "Why is it necessary to normalize all continuous variables before performing Lasso? (OPTIONAL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre> WRITE RESPONSE HERE </pre>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conclusions (2 marks)\n",
    "What values of $\\alpha$ make Lasso perform best? Does it perform as well on the housing data as the linear regressor from the lectures? Why do you think this is?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre> WRITE RESPONSE HERE </pre>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read appendix B\n",
    "\n",
    "- Reflect on your last data project, read appendix B. Then, write down a few of the checklist items that your last data project could have used. If you have not yet done a data project, then write down a few of the items that you found most interesting.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#YOUR ANSWER HERE"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}