mean_ensemble.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import numpy.ma as ma\n",
    "import pandas as pd\n",
    "\n",
    "from utils.predictorjpg import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The purpose of this notebook is to emsembling existing kaggle submission files by simple averaging and then apply thresholding to optimize F2 score.  \n",
    "\n",
    "The highest LB (F2) score for a single model (resnet50) is 0.92912.  With ensembling, we get 0.93146 which is an improvement of 0.00897.\n",
    "\n",
    "Due to lack of time to rerun training, the thresholding is approximate by averaging across thresholds for each model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_dir = 'D:/Downloads/amazon/'\n",
    "sample_submission_filepath = data_dir + 'sample_submission_v2.csv'\n",
    "real_submission_filepath = data_dir + 'my_submissions/ensemble/mean_ensemble_submission.csv'\n",
    "\n",
    "labels = ['slash_burn', 'clear', 'blooming', 'primary', 'cloudy', 'conventional_mine', 'water', 'haze', 'cultivation', 'partly_cloudy', 'artisinal_mine', 'habitation', 'bare_ground', 'blow_down', 'agriculture', 'road', 'selective_logging']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(61191L, 17L)\n"
     ]
    }
   ],
   "source": [
    "submit_dir = data_dir + 'my_submissions/'\n",
    "\n",
    "vgg19 = np.load(submit_dir + 'tta_submission_vgg19_20170719-113132.csv.h5.npy')\n",
    "resnet50 = np.load(submit_dir + 'tta_submission_resnet50_20170718-085651.csv.h5.npy')\n",
    "vgg16 = np.load(submit_dir + 'tta_submission_vgg16_20170720-065517.csv.h5.npy')\n",
    "#vgg16 = np.load(submit_dir + 'tta_submission_vgg16_20170717-171015.csv.h5.npy')\n",
    "densenet121 = np.load(submit_dir + 'tta_submission_densenet121_20170717-161234.csv.h5.npy')\n",
    "\n",
    "aggregation = []\n",
    "aggregation.append(vgg19)\n",
    "aggregation.append(resnet50)\n",
    "aggregation.append(vgg16)\n",
    "aggregation.append(densenet121)\n",
    "\n",
    "avg = np.mean(aggregation, axis=0)\n",
    "print(avg.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.3275  0.09    0.255   0.225   0.115   0.2725  0.1825  0.3175  0.2325\n",
      "  0.095   0.095   0.1725  0.195   0.145   0.1975  0.175   0.3125]\n"
     ]
    }
   ],
   "source": [
    "# these threshold number are determined by experiments carried out after individual model training.\n",
    "a = [0.29, 0.08, 0.31, 0.17, 0.04, 0.25, 0.14, 0.17, 0.21, 0.11, 0.14, 0.17, 0.15, 0.21, 0.20, 0.18, 0.12]\n",
    "b = [0.26, 0.12, 0.26, 0.21, 0.05, 0.20, 0.24, 0.29, 0.20, 0.05, 0.04, 0.14, 0.18, 0.12, 0.19, 0.13, 0.25]\n",
    "c = [0.12, 0.07, 0.15, 0.32, 0.12, 0.16, 0.13, 0.36, 0.27, 0.11, 0.14, 0.15, 0.17, 0.06, 0.16, 0.16, 0.46]\n",
    "d = [0.64, 0.09, 0.30, 0.20, 0.25, 0.48, 0.22, 0.45, 0.25, 0.11, 0.06, 0.23, 0.28, 0.19, 0.24, 0.23, 0.42]\n",
    "\n",
    "agg_threshold = []\n",
    "agg_threshold.append(a)\n",
    "agg_threshold.append(b)\n",
    "agg_threshold.append(c)\n",
    "agg_threshold.append(d)\n",
    "avg_threshold = np.mean(agg_threshold, axis=0)\n",
    "print(avg_threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 61191/61191 [01:03<00:00, 962.23it/s]\n"
     ]
    }
   ],
   "source": [
    "binary_predictions = (np.array(avg) > avg_threshold).astype(int)\n",
    "predict_df = pd.DataFrame(binary_predictions, columns = labels)\n",
    "\n",
    "df_submission = pd.read_csv(sample_submission_filepath)\n",
    "submit_df = submission_dataframe(df_submission, predict_df)\n",
    "submit_df.to_csv(real_submission_filepath, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}