autofeat_benchmark_classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import warnings\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.datasets import load_iris, load_wine, load_breast_cancer\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "from autofeat import AutoFeatClassifier\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = [\"iris\", \"wine\", \"breast_cancer\"]\n",
    "\n",
    "\n",
    "# same interface for loading all datasets\n",
    "def load_classification_dataset(name):\n",
    "    # load one of the datasets as X and y\n",
    "    units = {}\n",
    "    if name == \"iris\":\n",
    "        # sklearn iris housing dataset\n",
    "        X, y = load_iris(return_X_y=True)\n",
    "\n",
    "    elif name == \"wine\":\n",
    "        # sklearn wine dataset\n",
    "        X, y = load_wine(return_X_y=True)\n",
    "\n",
    "    elif name == \"breast_cancer\":\n",
    "        # sklearn breast_cancer dataset\n",
    "        X, y = load_breast_cancer(return_X_y=True)\n",
    "\n",
    "    else:\n",
    "        raise RuntimeError(f\"Unknown dataset {name}\")\n",
    "    return np.array(X, dtype=float), np.array(y, dtype=float), units\n",
    "\n",
    "\n",
    "def test_model(dataset, model, param_grid):\n",
    "    # load data\n",
    "    X, y, _ = load_classification_dataset(dataset)\n",
    "    # split in training and test parts\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=12)\n",
    "    if model.__class__.__name__ == \"SVC\":\n",
    "        sscaler = StandardScaler()\n",
    "        X_train = sscaler.fit_transform(X_train)\n",
    "        X_test = sscaler.transform(X_test)\n",
    "    # train model on train split incl cross-validation for parameter selection\n",
    "    with warnings.catch_warnings():\n",
    "        warnings.simplefilter(\"ignore\")\n",
    "        gsmodel = GridSearchCV(model, param_grid, cv=5)\n",
    "        gsmodel.fit(X_train, y_train)\n",
    "    print(\"best params:\", gsmodel.best_params_)\n",
    "    print(\"best score:\", gsmodel.best_score_)\n",
    "    print(\"Acc. on training data:\", accuracy_score(y_train, gsmodel.predict(X_train)))\n",
    "    print(\"Acc. on test data:\", accuracy_score(y_test, gsmodel.predict(X_test)))\n",
    "    return gsmodel.best_estimator_\n",
    "\n",
    "\n",
    "def test_autofeat(dataset, feateng_steps=2):\n",
    "    # load data\n",
    "    X, y, units = load_classification_dataset(dataset)\n",
    "    # split in training and test parts\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=12)\n",
    "    # run autofeat\n",
    "    afreg = AutoFeatClassifier(verbose=1, feateng_steps=feateng_steps, units=units)\n",
    "    # fit autofeat on less data, otherwise ridge reg model with xval will overfit on new features\n",
    "    X_train_tr = afreg.fit_transform(X_train, y_train)\n",
    "    X_test_tr = afreg.transform(X_test)\n",
    "    print(\"autofeat new features:\", len(afreg.new_feat_cols_))\n",
    "    print(\"autofeat Acc. on training data:\", accuracy_score(y_train, afreg.predict(X_train_tr)))\n",
    "    print(\"autofeat Acc. on test data:\", accuracy_score(y_test, afreg.predict(X_test_tr)))\n",
    "    # train rreg on transformed train split incl cross-validation for parameter selection\n",
    "    print(\"# Logistic Regression\")\n",
    "    rreg = LogisticRegression(class_weight=\"balanced\")\n",
    "    param_grid = {\"C\": np.logspace(-4, 4, 10)}\n",
    "    with warnings.catch_warnings():\n",
    "        warnings.simplefilter(\"ignore\")\n",
    "        gsmodel = GridSearchCV(rreg, param_grid, cv=5)\n",
    "        gsmodel.fit(X_train_tr, y_train)\n",
    "    print(\"best params:\", gsmodel.best_params_)\n",
    "    print(\"best score:\", gsmodel.best_score_)\n",
    "    print(\"Acc. on training data:\", accuracy_score(y_train, gsmodel.predict(X_train_tr)))\n",
    "    print(\"Acc. on test data:\", accuracy_score(y_test, gsmodel.predict(X_test_tr)))\n",
    "    print(\"# Random Forest\")\n",
    "    rforest = RandomForestClassifier(n_estimators=100, random_state=13)\n",
    "    param_grid = {\"min_samples_leaf\": [0.0001, 0.001, 0.01, 0.05, 0.1, 0.2]}\n",
    "    gsmodel = GridSearchCV(rforest, param_grid, cv=5)\n",
    "    gsmodel.fit(X_train_tr, y_train)\n",
    "    print(\"best params:\", gsmodel.best_params_)\n",
    "    print(\"best score:\", gsmodel.best_score_)\n",
    "    print(\"Acc. on training data:\", accuracy_score(y_train, gsmodel.predict(X_train_tr)))\n",
    "    print(\"Acc. on test data:\", accuracy_score(y_test, gsmodel.predict(X_test_tr)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dsname in datasets:\n",
    "    print(\"####\", dsname)\n",
    "    X, y, _ = load_classification_dataset(dsname)\n",
    "    print(X.shape, np.unique(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dsname in datasets:\n",
    "    print(\"####\", dsname)\n",
    "    rreg = LogisticRegression(class_weight=\"balanced\")\n",
    "    params = {\"C\": np.logspace(-4, 4, 10)}\n",
    "    rreg = test_model(dsname, rreg, params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dsname in datasets:\n",
    "    print(\"####\", dsname)\n",
    "    svc = SVC(gamma=\"scale\", class_weight=\"balanced\")\n",
    "    params = {\"C\": [1.0, 10.0, 25.0, 50.0, 100.0, 250.0]}\n",
    "    svc = test_model(dsname, svc, params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dsname in datasets:\n",
    "    print(\"####\", dsname)\n",
    "    rforest = RandomForestClassifier(n_estimators=100, random_state=13)\n",
    "    params = {\"min_samples_leaf\": [0.0001, 0.001, 0.01, 0.05, 0.1, 0.2]}\n",
    "    rforest = test_model(dsname, rforest, params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dsname in datasets:\n",
    "    print(\"####\", dsname)\n",
    "    test_autofeat(dsname, feateng_steps=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}