Experiment_prepare_data_normalized_group.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. first reproduced the high scores\n",
    "2. try to use normalization accross samples and see if this can help increase the score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.ensemble import RandomForestClassifier\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_onehot(y_ex):\n",
    "    label_encoder = LabelEncoder()\n",
    "    integer_encoded = label_encoder.fit_transform(y_ex.astype('category'))\n",
    "#     # binary encode\n",
    "#     onehot_encoder = OneHotEncoder(sparse=False, categories='auto')\n",
    "#     integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n",
    "#     y_coded = onehot_encoder.fit_transform(integer_encoded)\n",
    "    \n",
    "    return (label_encoder, integer_encoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_prediction_kfold(test, data, y):\n",
    "    \n",
    "    le, target = convert_onehot(y.surface)\n",
    "    target = target.ravel()\n",
    "    print (np.shape(target))\n",
    "\n",
    "    n_class = 9\n",
    "    \n",
    "    folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=20)\n",
    "    predicted = np.zeros((test.shape[0],n_class))\n",
    "    measured= np.zeros((data.shape[0]))\n",
    "    score = 0\n",
    "    \n",
    "    print (np.shape(test))\n",
    "    print (np.shape(data))\n",
    "\n",
    "    for times, (trn_idx, val_idx) in enumerate(folds.split(data.values,target)):\n",
    "        model = RandomForestClassifier(n_estimators=500, n_jobs = -1)\n",
    "        model.fit(data.iloc[trn_idx],target[trn_idx])\n",
    "        measured[val_idx] = model.predict(data.iloc[val_idx])\n",
    "        predicted += model.predict_proba(test)/folds.n_splits\n",
    "        score += model.score(data.iloc[val_idx],target[val_idx])\n",
    "        print(\"Fold: {} score: {}\".format(times,model.score(data.iloc[val_idx],target[val_idx])))\n",
    "        gc.collect()\n",
    "    \n",
    "    result=le.inverse_transform(predicted.argmax(axis=1))\n",
    "    result_pd = pd.DataFrame({'series_id': list(range(len(result))), 'surface':result})\n",
    "                              \n",
    "    return (result_pd)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_all = pd.read_csv('New_data\\X_test_selected_features.csv')\n",
    "X_train_all = pd.read_csv('New_data\\X_train_selected_features.csv')\n",
    "y_train_all = pd.read_csv('New_data\\Train_y.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>orientation_Z__cwt_coefficients__widths_(2, 5, 10, 20)__coeff_9__w_10</th>\n",
       "      <th>linear_acceleration_Z__partial_autocorrelation__lag_4</th>\n",
       "      <th>orientation_W__agg_linear_trend__f_agg_\"var\"__chunk_len_5__attr_\"rvalue\"</th>\n",
       "      <th>orientation_W__fft_coefficient__coeff_21__attr_\"real\"</th>\n",
       "      <th>orientation_Z__fft_coefficient__coeff_27__attr_\"real\"</th>\n",
       "      <th>angular_velocity_X__longest_strike_above_mean</th>\n",
       "      <th>orientation_X__fft_coefficient__coeff_19__attr_\"imag\"</th>\n",
       "      <th>angle_Y__cwt_coefficients__widths_(2, 5, 10, 20)__coeff_1__w_20</th>\n",
       "      <th>orientation_Y__fft_coefficient__coeff_53__attr_\"abs\"</th>\n",
       "      <th>...</th>\n",
       "      <th>angle_Z__abs_energy</th>\n",
       "      <th>linear_acceleration_X__partial_autocorrelation__lag_6</th>\n",
       "      <th>orientation_W__abs_energy</th>\n",
       "      <th>angular_velocity_X__fft_aggregated__aggtype_\"centroid\"</th>\n",
       "      <th>orientation_X__cwt_coefficients__widths_(2, 5, 10, 20)__coeff_0__w_5</th>\n",
       "      <th>angle_Y__fft_coefficient__coeff_21__attr_\"abs\"</th>\n",
       "      <th>orientation_W__partial_autocorrelation__lag_8</th>\n",
       "      <th>angle_X__fft_coefficient__coeff_36__attr_\"abs\"</th>\n",
       "      <th>orientation_Y__fft_coefficient__coeff_57__attr_\"real\"</th>\n",
       "      <th>orientation_W__agg_linear_trend__f_agg_\"mean\"__chunk_len_5__attr_\"stderr\"</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.243516</td>\n",
       "      <td>-0.128970</td>\n",
       "      <td>-0.271917</td>\n",
       "      <td>-0.000634</td>\n",
       "      <td>-0.000660</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.000481</td>\n",
       "      <td>-0.005533</td>\n",
       "      <td>0.000198</td>\n",
       "      <td>...</td>\n",
       "      <td>1223.315786</td>\n",
       "      <td>0.070472</td>\n",
       "      <td>0.001275</td>\n",
       "      <td>22.620098</td>\n",
       "      <td>-0.000007</td>\n",
       "      <td>0.001437</td>\n",
       "      <td>-0.477185</td>\n",
       "      <td>0.000207</td>\n",
       "      <td>0.000088</td>\n",
       "      <td>0.000006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.070733</td>\n",
       "      <td>-0.117502</td>\n",
       "      <td>-0.332469</td>\n",
       "      <td>-0.001564</td>\n",
       "      <td>0.001028</td>\n",
       "      <td>18.0</td>\n",
       "      <td>-0.002241</td>\n",
       "      <td>-0.004646</td>\n",
       "      <td>0.002857</td>\n",
       "      <td>...</td>\n",
       "      <td>59.034651</td>\n",
       "      <td>-0.173276</td>\n",
       "      <td>2.543646</td>\n",
       "      <td>16.680664</td>\n",
       "      <td>0.000177</td>\n",
       "      <td>0.000281</td>\n",
       "      <td>-0.462603</td>\n",
       "      <td>0.002491</td>\n",
       "      <td>0.002807</td>\n",
       "      <td>0.000015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.238500</td>\n",
       "      <td>0.016427</td>\n",
       "      <td>-0.225892</td>\n",
       "      <td>0.001418</td>\n",
       "      <td>-0.000237</td>\n",
       "      <td>5.0</td>\n",
       "      <td>-0.003672</td>\n",
       "      <td>-0.005516</td>\n",
       "      <td>0.000322</td>\n",
       "      <td>...</td>\n",
       "      <td>912.033423</td>\n",
       "      <td>0.002008</td>\n",
       "      <td>0.230062</td>\n",
       "      <td>22.401308</td>\n",
       "      <td>0.001558</td>\n",
       "      <td>0.001431</td>\n",
       "      <td>-0.406308</td>\n",
       "      <td>0.001419</td>\n",
       "      <td>0.000596</td>\n",
       "      <td>0.000008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.241586</td>\n",
       "      <td>0.200422</td>\n",
       "      <td>0.131696</td>\n",
       "      <td>-0.000491</td>\n",
       "      <td>0.000196</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.004283</td>\n",
       "      <td>-0.003659</td>\n",
       "      <td>0.000342</td>\n",
       "      <td>...</td>\n",
       "      <td>1009.975832</td>\n",
       "      <td>0.320353</td>\n",
       "      <td>0.041927</td>\n",
       "      <td>20.748559</td>\n",
       "      <td>-0.000111</td>\n",
       "      <td>0.000577</td>\n",
       "      <td>-0.013068</td>\n",
       "      <td>0.000638</td>\n",
       "      <td>0.000293</td>\n",
       "      <td>0.000002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.235623</td>\n",
       "      <td>-0.031627</td>\n",
       "      <td>-0.590452</td>\n",
       "      <td>0.002659</td>\n",
       "      <td>0.000978</td>\n",
       "      <td>29.0</td>\n",
       "      <td>-0.032946</td>\n",
       "      <td>-0.002930</td>\n",
       "      <td>0.004485</td>\n",
       "      <td>...</td>\n",
       "      <td>879.387663</td>\n",
       "      <td>0.010791</td>\n",
       "      <td>0.248844</td>\n",
       "      <td>14.112171</td>\n",
       "      <td>0.002739</td>\n",
       "      <td>0.000528</td>\n",
       "      <td>-0.015644</td>\n",
       "      <td>0.000534</td>\n",
       "      <td>0.004363</td>\n",
       "      <td>0.000002</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 4990 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  orientation_Z__cwt_coefficients__widths_(2, 5, 10, 20)__coeff_9__w_10  \\\n",
       "0   0                                          -0.243516                       \n",
       "1   1                                           0.070733                       \n",
       "2   2                                           0.238500                       \n",
       "3   3                                           0.241586                       \n",
       "4   4                                           0.235623                       \n",
       "\n",
       "   linear_acceleration_Z__partial_autocorrelation__lag_4  \\\n",
       "0                                          -0.128970       \n",
       "1                                          -0.117502       \n",
       "2                                           0.016427       \n",
       "3                                           0.200422       \n",
       "4                                          -0.031627       \n",
       "\n",
       "   orientation_W__agg_linear_trend__f_agg_\"var\"__chunk_len_5__attr_\"rvalue\"  \\\n",
       "0                                          -0.271917                          \n",
       "1                                          -0.332469                          \n",
       "2                                          -0.225892                          \n",
       "3                                           0.131696                          \n",
       "4                                          -0.590452                          \n",
       "\n",
       "   orientation_W__fft_coefficient__coeff_21__attr_\"real\"  \\\n",
       "0                                          -0.000634       \n",
       "1                                          -0.001564       \n",
       "2                                           0.001418       \n",
       "3                                          -0.000491       \n",
       "4                                           0.002659       \n",
       "\n",
       "   orientation_Z__fft_coefficient__coeff_27__attr_\"real\"  \\\n",
       "0                                          -0.000660       \n",
       "1                                           0.001028       \n",
       "2                                          -0.000237       \n",
       "3                                           0.000196       \n",
       "4                                           0.000978       \n",
       "\n",
       "   angular_velocity_X__longest_strike_above_mean  \\\n",
       "0                                            5.0   \n",
       "1                                           18.0   \n",
       "2                                            5.0   \n",
       "3                                            4.0   \n",
       "4                                           29.0   \n",
       "\n",
       "   orientation_X__fft_coefficient__coeff_19__attr_\"imag\"  \\\n",
       "0                                           0.000481       \n",
       "1                                          -0.002241       \n",
       "2                                          -0.003672       \n",
       "3                                           0.004283       \n",
       "4                                          -0.032946       \n",
       "\n",
       "   angle_Y__cwt_coefficients__widths_(2, 5, 10, 20)__coeff_1__w_20  \\\n",
       "0                                          -0.005533                 \n",
       "1                                          -0.004646                 \n",
       "2                                          -0.005516                 \n",
       "3                                          -0.003659                 \n",
       "4                                          -0.002930                 \n",
       "\n",
       "   orientation_Y__fft_coefficient__coeff_53__attr_\"abs\"  \\\n",
       "0                                           0.000198      \n",
       "1                                           0.002857      \n",
       "2                                           0.000322      \n",
       "3                                           0.000342      \n",
       "4                                           0.004485      \n",
       "\n",
       "                                     ...                                      \\\n",
       "0                                    ...                                       \n",
       "1                                    ...                                       \n",
       "2                                    ...                                       \n",
       "3                                    ...                                       \n",
       "4                                    ...                                       \n",
       "\n",
       "   angle_Z__abs_energy  linear_acceleration_X__partial_autocorrelation__lag_6  \\\n",
       "0          1223.315786                                           0.070472       \n",
       "1            59.034651                                          -0.173276       \n",
       "2           912.033423                                           0.002008       \n",
       "3          1009.975832                                           0.320353       \n",
       "4           879.387663                                           0.010791       \n",
       "\n",
       "   orientation_W__abs_energy  \\\n",
       "0                   0.001275   \n",
       "1                   2.543646   \n",
       "2                   0.230062   \n",
       "3                   0.041927   \n",
       "4                   0.248844   \n",
       "\n",
       "   angular_velocity_X__fft_aggregated__aggtype_\"centroid\"  \\\n",
       "0                                          22.620098        \n",
       "1                                          16.680664        \n",
       "2                                          22.401308        \n",
       "3                                          20.748559        \n",
       "4                                          14.112171        \n",
       "\n",
       "   orientation_X__cwt_coefficients__widths_(2, 5, 10, 20)__coeff_0__w_5  \\\n",
       "0                                          -0.000007                      \n",
       "1                                           0.000177                      \n",
       "2                                           0.001558                      \n",
       "3                                          -0.000111                      \n",
       "4                                           0.002739                      \n",
       "\n",
       "   angle_Y__fft_coefficient__coeff_21__attr_\"abs\"  \\\n",
       "0                                        0.001437   \n",
       "1                                        0.000281   \n",
       "2                                        0.001431   \n",
       "3                                        0.000577   \n",
       "4                                        0.000528   \n",
       "\n",
       "   orientation_W__partial_autocorrelation__lag_8  \\\n",
       "0                                      -0.477185   \n",
       "1                                      -0.462603   \n",
       "2                                      -0.406308   \n",
       "3                                      -0.013068   \n",
       "4                                      -0.015644   \n",
       "\n",
       "   angle_X__fft_coefficient__coeff_36__attr_\"abs\"  \\\n",
       "0                                        0.000207   \n",
       "1                                        0.002491   \n",
       "2                                        0.001419   \n",
       "3                                        0.000638   \n",
       "4                                        0.000534   \n",
       "\n",
       "   orientation_Y__fft_coefficient__coeff_57__attr_\"real\"  \\\n",
       "0                                           0.000088       \n",
       "1                                           0.002807       \n",
       "2                                           0.000596       \n",
       "3                                           0.000293       \n",
       "4                                           0.004363       \n",
       "\n",
       "   orientation_W__agg_linear_trend__f_agg_\"mean\"__chunk_len_5__attr_\"stderr\"  \n",
       "0                                           0.000006                          \n",
       "1                                           0.000015                          \n",
       "2                                           0.000008                          \n",
       "3                                           0.000002                          \n",
       "4                                           0.000002                          \n",
       "\n",
       "[5 rows x 4990 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4021,)\n",
      "(3816, 4989)\n",
      "(4021, 4989)\n",
      "Fold: 0 score: 0.9187192118226601\n",
      "Fold: 1 score: 0.8793103448275862\n",
      "Fold: 2 score: 0.9158415841584159\n",
      "Fold: 3 score: 0.9280397022332506\n",
      "Fold: 4 score: 0.8980099502487562\n",
      "Fold: 5 score: 0.9104477611940298\n",
      "Fold: 6 score: 0.8805970149253731\n",
      "Fold: 7 score: 0.905\n",
      "Fold: 8 score: 0.9122807017543859\n",
      "Fold: 9 score: 0.9219143576826196\n"
     ]
    }
   ],
   "source": [
    "result = make_prediction_kfold(X_test_all.drop(['id'], axis = 1), X_train_all.drop(['id'], axis = 1), y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "result.to_csv(r'New_data\\results\\RF_500_check_20190404.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# yes, the resutl is the same 0.68"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_prediction_kfold_norm(test, data, y):\n",
    "    \n",
    "    le, target = convert_onehot(y.surface)\n",
    "    target = target.ravel()\n",
    "    print (np.shape(target))\n",
    "\n",
    "    n_class = 9\n",
    "    \n",
    "    ## normalize the data\n",
    "    data = normalize(data)\n",
    "    test =normalize(test)\n",
    "    \n",
    "    folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=20)\n",
    "    predicted = np.zeros((test.shape[0],n_class))\n",
    "    measured= np.zeros((data.shape[0]))\n",
    "    score = 0\n",
    "    \n",
    "    print (np.shape(test))\n",
    "    print (np.shape(data))\n",
    "\n",
    "    for times, (trn_idx, val_idx) in enumerate(folds.split(data,target)):\n",
    "        model = RandomForestClassifier(n_estimators=500, n_jobs = -1)\n",
    "        model.fit(data[trn_idx,:],target[trn_idx])\n",
    "        measured[val_idx] = model.predict(data[val_idx,:])\n",
    "        predicted += model.predict_proba(test)/folds.n_splits\n",
    "        score += model.score(data[val_idx,:],target[val_idx])\n",
    "        print(\"Fold: {} score: {}\".format(times,model.score(data[val_idx,:],target[val_idx])))\n",
    "        gc.collect()\n",
    "    \n",
    "    result=le.inverse_transform(predicted.argmax(axis=1))\n",
    "    result_pd = pd.DataFrame({'series_id': list(range(len(result))), 'surface':result})\n",
    "                              \n",
    "    return (result_pd)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4021,)\n",
      "(3816, 4989)\n",
      "(4021, 4989)\n",
      "Fold: 0 score: 0.7536945812807881\n",
      "Fold: 1 score: 0.687192118226601\n",
      "Fold: 2 score: 0.7128712871287128\n",
      "Fold: 3 score: 0.7444168734491315\n",
      "Fold: 4 score: 0.6890547263681592\n",
      "Fold: 5 score: 0.7164179104477612\n",
      "Fold: 6 score: 0.6940298507462687\n",
      "Fold: 7 score: 0.71\n",
      "Fold: 8 score: 0.7042606516290727\n",
      "Fold: 9 score: 0.7581863979848866\n"
     ]
    }
   ],
   "source": [
    "result_norm = make_prediction_kfold_norm(X_test_all.drop(['id'], axis = 1), X_train_all.drop(['id'], axis = 1), y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_norm.to_csv(r'New_data\\results\\RF_500_check_norm_data_20190404.csv', index=False)\n",
    "# score 0.52"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## use the whole data set in training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_prediction_kfold_norm_whole_training_dataset(test, data, y):\n",
    "    \n",
    "    le, target = convert_onehot(y.surface)\n",
    "    target = target.ravel()\n",
    "    print (np.shape(target))\n",
    "\n",
    "    n_class = 9\n",
    "    \n",
    "    ## normalize the data\n",
    "    data = normalize(data)\n",
    "    test =normalize(test)\n",
    "    \n",
    "    print (np.shape(test))\n",
    "    print (np.shape(data))\n",
    "\n",
    "    model = RandomForestClassifier(n_estimators=500, n_jobs = -1)\n",
    "    model.fit(data,target)\n",
    "    predicted = model.predict(test)\n",
    "    \n",
    "    result=le.inverse_transform(predicted)\n",
    "    result_pd = pd.DataFrame({'series_id': list(range(len(result))), 'surface':result})\n",
    "                              \n",
    "    return (result_pd)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4021,)\n",
      "(3816, 4989)\n",
      "(4021, 4989)\n"
     ]
    }
   ],
   "source": [
    "result_norm_whole = make_prediction_kfold_norm_whole_training_dataset(X_test_all.drop(['id'], axis = 1), \\\n",
    "                                                                X_train_all.drop(['id'], axis = 1), y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_norm_whole.to_csv(r'New_data\\results\\RF_500_check_norm_data_train_whole_data_20190404.csv', index=False)\n",
    "# also score 0.52"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}