binary_voting_ensembling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n",
      "Using cuDNN version 5110 on context None\n",
      "Mapped name None to device cuda: GeForce GTX 1060 6GB (0000:01:00.0)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import numpy.ma as ma\n",
    "import pandas as pd\n",
    "\n",
    "from keras.models import load_model\n",
    "\n",
    "from utils.ensemble import *\n",
    "from utils.f2thresholdfinder import *\n",
    "from utils.loaderjpg import *\n",
    "from utils.generator import *\n",
    "\n",
    "from pretrained.custommodels import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The purose of this notebook is to find the optimal weight distributions (voting power) to use in ensembling results from non-correlated models. Ensembling is done via voting where one model can have more voting power than another model.  I hand picked the best models to be used in ensembling.  Since the F2 calculation is super fast, we use a brute force approach to find the optimal weights that will optimize a metrics (in this case, the F2 score)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40479L, 224L, 224L, 3L)\n",
      "(40479L, 17L)\n"
     ]
    }
   ],
   "source": [
    "rescaled_dim = 224\n",
    "data_dir = 'D:/Downloads/amazon/'\n",
    "\n",
    "df_train = pd.read_csv(data_dir + 'train_v2.csv')\n",
    "x_train, y_train = load_training_set(df_train, rescaled_dim)\n",
    "print(x_train.shape)\n",
    "print(y_train.shape)\n",
    "\n",
    "number_of_samples = x_train.shape[0]\n",
    "split = int(number_of_samples * 0.90)\n",
    "                     \n",
    "x_valid, y_valid = x_train[split:], y_train[split:]\n",
    "\n",
    "del x_train, y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def predict_from_model(x, y, model_name, model):\n",
    "    img_normalization = image_normalization_func(model_name)\n",
    "    generator = BottleNeckImgGenerator(normalization=img_normalization)\n",
    "    y_predict, thresholds = predict_with_optimal_thresholds(x, y, generator, model)\n",
    "    return y_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Me\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:1113: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label:0 threshold:0.29 score:0.901941455181\n",
      "label:1 threshold:0.08 score:0.90539498334\n",
      "label:2 threshold:0.31 score:0.905609688139\n",
      "label:3 threshold:0.17 score:0.907871945561\n",
      "label:4 threshold:0.04 score:0.911936839873\n",
      "label:5 threshold:0.25 score:0.911977083764\n",
      "label:6 threshold:0.14 score:0.914077703266\n",
      "label:7 threshold:0.17 score:0.916296101521\n",
      "label:8 threshold:0.21 score:0.919021713075\n",
      "label:9 threshold:0.11 score:0.920021282337\n",
      "label:10 threshold:0.14 score:0.92023644765\n",
      "label:11 threshold:0.17 score:0.922349162987\n",
      "label:12 threshold:0.15 score:0.923297033475\n",
      "label:13 threshold:0.21 score:0.92330687569\n",
      "label:14 threshold:0.2 score:0.927807214006\n",
      "label:15 threshold:0.18 score:0.930156605232\n",
      "label:16 threshold:0.12 score:0.930250803913\n",
      "('>>>> Overall precision score over validation set ', 0.87324594940084077)\n",
      "('>>>> Overall recall score over validation set ', 0.95798483672124968)\n",
      "('>>>> Overall F2 score over validation set ', 0.93025080391310422)\n",
      "label:0 threshold:0.26 score:0.9010781626\n",
      "label:1 threshold:0.12 score:0.905239313339\n",
      "label:2 threshold:0.26 score:0.905523334487\n",
      "label:3 threshold:0.21 score:0.907586584475\n",
      "label:4 threshold:0.05 score:0.913399055916\n",
      "label:5 threshold:0.2 score:0.913429534331\n",
      "label:6 threshold:0.24 score:0.915529668249\n",
      "label:7 threshold:0.29 score:0.916536361744\n",
      "label:8 threshold:0.2 score:0.919284178811\n",
      "label:9 threshold:0.05 score:0.92040775337\n",
      "label:10 threshold:0.04 score:0.920597313982\n",
      "label:11 threshold:0.14 score:0.922680070835\n",
      "label:12 threshold:0.18 score:0.923829673004\n",
      "label:13 threshold:0.12 score:0.924152134005\n",
      "label:14 threshold:0.19 score:0.928874002639\n",
      "label:15 threshold:0.13 score:0.932059600726\n",
      "label:16 threshold:0.25 score:0.932172400117\n",
      "('>>>> Overall precision score over validation set ', 0.87611411082878465)\n",
      "('>>>> Overall recall score over validation set ', 0.95902826792772444)\n",
      "('>>>> Overall F2 score over validation set ', 0.93217240011738645)\n",
      "label:0 threshold:0.48 score:0.905264539056\n",
      "label:1 threshold:0.08 score:0.908145071858\n",
      "label:2 threshold:0.22 score:0.908243252557\n",
      "label:3 threshold:0.21 score:0.912988307192\n",
      "label:4 threshold:0.26 score:0.915594935507\n",
      "label:5 threshold:0.57 score:0.915613200042\n",
      "label:6 threshold:0.25 score:0.917620833644\n",
      "label:7 threshold:0.36 score:0.918202580441\n",
      "label:8 threshold:0.25 score:0.92036033415\n",
      "label:9 threshold:0.11 score:0.922617924549\n",
      "label:10 threshold:0.09 score:0.922826744556\n",
      "label:11 threshold:0.21 score:0.924346456657\n",
      "label:12 threshold:0.24 score:0.924878787554\n",
      "label:13 threshold:0.12 score:0.925093780376\n",
      "label:14 threshold:0.18 score:0.929396044556\n",
      "label:15 threshold:0.24 score:0.930340278193\n",
      "label:16 threshold:0.59 score:0.930405251135\n",
      "('>>>> Overall precision score over validation set ', 0.87298940491875276)\n",
      "('>>>> Overall recall score over validation set ', 0.95741930171277989)\n",
      "('>>>> Overall F2 score over validation set ', 0.93040525113487682)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "y_predictions = []\n",
    "\n",
    "model_name = 'resnet50'\n",
    "model_filepath = None\n",
    "weights_filepath = 'D:/Downloads/amazon/bottleneck/resnet50/frozen38_20170717-092206_weights_only.h5'\n",
    "model = custom_top_model(model_name, num_classes=17, num_frozen_layers=0)\n",
    "model.load_weights(weights_filepath)\n",
    "img_normalization = image_normalization_func(model_name)\n",
    "y_predictions.append(predict_from_model(x_valid, y_valid, model_name, model))\n",
    "\n",
    "model_name = 'densenet121'\n",
    "model_filepath = None\n",
    "weights_filepath = 'D:/Downloads/amazon/bottleneck/densenet121/frozen73_20170716-171641_weights_only.h5'\n",
    "model = custom_top_model(model_name, num_classes=17, num_frozen_layers=0)\n",
    "model.load_weights(weights_filepath)\n",
    "y_predictions.append(predict_from_model(x_valid, y_valid, model_name, model))\n",
    "\n",
    "# vgg19\n",
    "vgg19_predictions = np.load('D:/Downloads/amazon/temp/vgg19_20170719-113132_valid_set_predictions.npy')\n",
    "y_predictions.append(vgg19_predictions)\n",
    "\n",
    "model_name = 'vgg16'\n",
    "model_filepath = 'D:/Downloads/amazon/bottleneck/vgg16/frozen11_20170706-011852.h5'\n",
    "weights_filepath = None\n",
    "model = load_model(model_filepath)\n",
    "y_predictions.append(predict_from_model(x_valid, y_valid, model_name, model))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ekami_predictions = np.load('D:/Downloads/amazon/temp/valid_set_predictions.npy')\n",
    "y_predictions.append(ekami_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weighted_ensemble_f2_score_optimizer4(weights_combo, y_valid, y_predictions):\n",
    "\t\"\"\"\n",
    "\t\tAn optimizer_func implementation that evaluate F2 scores as the metrics to optimize.\n",
    "\t\"\"\"\n",
    "\ty_predict_aggregate = np.zeros((y_valid.shape[0], y_valid.shape[1]), dtype=np.float32)\n",
    "\tfor weight, y_predict in zip(weights_combo, y_predictions):\n",
    "\t\ty_predict_aggregate = y_predict_aggregate + (y_predict.astype(np.float32) * weight)\n",
    "\tbinary_predictions = (np.array(y_predict_aggregate) > 0.60).astype(int)\n",
    "\tf2_score = fbeta_score(y_valid, binary_predictions, beta=2, average='samples')\n",
    "\tprint('> F2 score : {} for weights: {}'.format(f2_score, weights_combo))\n",
    "\treturn f2_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> F2 score : 0.935239052542 for weights: [0.2, 0.2, 0.2, 0.2, 0.2]\n",
      "> F2 score : 0.930250803913 for weights: [0.5, 0.1, 0.1, 0.1, 0.1]\n",
      "> F2 score : 0.934793041667 for weights: [0.4, 0.3, 0.2, 0.1, 0.0]\n",
      "> F2 score : 0.934345546259 for weights: [0.4, 0.3, 0.3, 0.0, 0.0]\n",
      "> F2 score : 0.934410244186 for weights: [0.4, 0.2, 0.2, 0.1, 0.1]\n",
      "> F2 score : 0.934556026963 for weights: [0.3, 0.2, 0.2, 0.1, 0.2]\n",
      "> F2 score : 0.934351991152 for weights: [0.25, 0.25, 0.25, 0.0, 0.25]\n",
      "> F2 score : 0.934473354748 for weights: [0.25, 0.25, 0.25, 0.25, 0.0]\n",
      "> F2 score : 0.934308568233 for weights: [0.3, 0.3, 0.2, 0.2, 0.0]\n",
      "> F2 score : 0.934900670009 for weights: [0.3, 0.2, 0.2, 0.2, 0.1]\n",
      "> F2 score : 0.935239052542 for weights: [0.25, 0.2, 0.2, 0.2, 0.15]\n",
      "> F2 score : 0.934664339738 for weights: [0.3, 0.3, 0.2, 0.1, 0.1]\n",
      "> F2 score : 0.934345546259 for weights: [0.3, 0.3, 0.25, 0.05, 0.1]\n",
      "> F2 score : 0.934808684268 for weights: [0.35, 0.25, 0.25, 0.05, 0.1]\n",
      "> F2 score : 0.934345546259 for weights: [0.333, 0.333, 0.333, 0.0, 0.0]\n",
      "> F2 score : 0.934708617241 for weights: [0.333, 0.0, 0.333, 0.333, 0.0]\n",
      "> F2 score : 0.934267925187 for weights: [0.333, 0.333, 0.0, 0.333, 0.0]\n",
      "> F2 score : 0.934708617241 for weights: [0.333, 0.0, 0.333, 0.333, 0.0]\n",
      "optimal_weights: [0.2, 0.2, 0.2, 0.2, 0.2]\n"
     ]
    }
   ],
   "source": [
    "weights_combos = [\n",
    "    [0.2, 0.2, 0.2, 0.2, 0.2],\n",
    "    [0.5, 0.1, 0.1, 0.1, 0.1],\n",
    "    [0.4, 0.3, 0.2, 0.1, 0.0],\n",
    "    [0.4, 0.3, 0.3, 0.0, 0.0],\n",
    "    [0.4, 0.2, 0.2, 0.1, 0.1],\n",
    "    [0.3, 0.2, 0.2, 0.1, 0.2],\n",
    "    [0.25, 0.25, 0.25, 0.0 ,0.25],\n",
    "    [0.25, 0.25, 0.25, 0.25,  0.0],\n",
    "    [0.3, 0.3, 0.2, 0.2,  0.0],\n",
    "    [0.3, 0.2, 0.2, 0.2, 0.1],\n",
    "    [0.25, 0.2, 0.2, 0.2, 0.15],\n",
    "    [0.3, 0.3, 0.2, 0.1, 0.1],\n",
    "    [0.3, 0.3, 0.25, 0.05, 0.1],\n",
    "    [0.35, 0.25, 0.25, 0.05, 0.1],\n",
    "    [0.333, 0.333, 0.333, 0.0, 0.0],\n",
    "    [0.333, 0.0, 0.333, 0.333, 0.0],\n",
    "    [0.333, 0.333, 0.0, 0.333, 0.0],\n",
    "    [0.333, 0.0, 0.333, 0.333, 0.0]\n",
    "]\n",
    "\n",
    "# weights_combos = [\n",
    "#     [0.3333, 0.3333, 0.3333, 0.0],\n",
    "#     [0.4, 0.2, 0.2, 0.2],\n",
    "#     [0.3333, 0.3333, 0.2333, 0.1],\n",
    "#     [0.3333, 0.2333, 0.2333, 0.2],\n",
    "#     [0.35, 0.30, 0.15, 0.20],\n",
    "#     [0.4, 0.4, 0, 0.2],\n",
    "#     [0.35, 0.35, 0, 0.3],\n",
    "#     [0.4, 0.4, 0.1, 0.1],\n",
    "#     [0.3,0.3,0.2,0.2],\n",
    "#     [0.35,0.35,0.1,0.2],\n",
    "#     [0.4,0.3,0.2,0.1],\n",
    "#     [0.35,0.35,0.15,0.15],\n",
    "#     [0.25,0.25,0.25,0.25],\n",
    "#     [0.3, 0.3, 0.1, 0.3],\n",
    "#     [0.4,0.3,0.2,0.1],\n",
    "#     [0.3,0.3,0.3,0.1],\n",
    "#     [0.3,0.25,0.25,0.2],\n",
    "#     [0.35,0.3,0.2,0.15],\n",
    "#     [0.35,0.35,0.15,0.15],\n",
    "#     [0.35,0.15,0.35,0.15],\n",
    "# ]\n",
    "\n",
    "# weights_combos = [\n",
    "#     [0.2, 0.5, 0.3],\n",
    "#     [0.3333, 0.3333, 0.3333],\n",
    "#     [0.4, 0.3, 0.3],\n",
    "#     [0.4, 0.35, 0.25],\n",
    "#     [0.4, 0.4, 0.2],\n",
    "#     [0.45, 0.35, 0.2],\n",
    "#     [0.5, 0.25, 0.25],\n",
    "#     [0.5, 0.3, 0.2],\n",
    "#     [0.6, 0.2, 0.2],\n",
    "#     [0.7, 0.2, 0.1],\n",
    "# ]\n",
    "\n",
    "optimal_weights = eval_optimal_ensemble_weights(weights_combos, \n",
    "                                                y_predictions,\n",
    "                                                y_valid, \n",
    "                                                weighted_ensemble_f2_score_optimizer)\n",
    "\n",
    "print('optimal_weights: {}'.format(optimal_weights))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ensembling kaggle submission files: ['tta_submission_resnet50_20170718-085651_score_092912.csv', 'tta_submission_densenet121_20170717-161234_score_092785.csv', 'tta_submission_vgg19_20170719-113132_score_092790.csv', 'tta_submission_vgg16_20170717-171015_score_092680.csv', 'submission_20170718-163707.csv']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 61191/61191 [00:58<00:00, 1047.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "submission file generated: D:/Downloads/amazon/my_submissions/ensemble/weighted_resnet50_densenet121_vgg19_vgg16_ekami_tta.csv\n"
     ]
    }
   ],
   "source": [
    "submission_files = [\n",
    "# WARNING!!!: ONLY USE FILES generated AFTER 6/25/2017 at 7pm (post Kaggle test data patch).\n",
    "    'tta_submission_resnet50_20170718-085651_score_092912.csv',\n",
    "    'tta_submission_densenet121_20170717-161234_score_092785.csv',\n",
    "    'tta_submission_vgg19_20170719-113132_score_092790.csv',\n",
    "    'tta_submission_vgg16_20170717-171015_score_092680.csv',\n",
    "    'submission_20170718-163707.csv' #'submission_20170626-025551_score_091200.csv'\n",
    "]\n",
    "\n",
    "generate_ensemble_submission('weighted_resnet50_densenet121_vgg19_vgg16_ekami_tta.csv', submission_files, optimal_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}