512-fitting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import keras\n",
    "import os\n",
    "import numpy as np\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read test data\n",
    "data = pd.read_csv(\"question_embeddings0.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>502</th>\n",
       "      <th>503</th>\n",
       "      <th>504</th>\n",
       "      <th>505</th>\n",
       "      <th>506</th>\n",
       "      <th>507</th>\n",
       "      <th>508</th>\n",
       "      <th>509</th>\n",
       "      <th>510</th>\n",
       "      <th>511</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.004167</td>\n",
       "      <td>0.003991</td>\n",
       "      <td>-0.063261</td>\n",
       "      <td>0.051590</td>\n",
       "      <td>-0.041000</td>\n",
       "      <td>-0.044420</td>\n",
       "      <td>0.084289</td>\n",
       "      <td>-0.007363</td>\n",
       "      <td>-0.021403</td>\n",
       "      <td>-0.022497</td>\n",
       "      <td>...</td>\n",
       "      <td>0.060429</td>\n",
       "      <td>-0.062331</td>\n",
       "      <td>-0.017937</td>\n",
       "      <td>-0.033932</td>\n",
       "      <td>0.009563</td>\n",
       "      <td>-0.063174</td>\n",
       "      <td>0.053774</td>\n",
       "      <td>-0.043223</td>\n",
       "      <td>-0.044161</td>\n",
       "      <td>-0.066338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.028851</td>\n",
       "      <td>-0.046769</td>\n",
       "      <td>-0.003425</td>\n",
       "      <td>0.002011</td>\n",
       "      <td>0.038640</td>\n",
       "      <td>0.033116</td>\n",
       "      <td>0.003396</td>\n",
       "      <td>0.041565</td>\n",
       "      <td>-0.004693</td>\n",
       "      <td>0.022073</td>\n",
       "      <td>...</td>\n",
       "      <td>0.045903</td>\n",
       "      <td>0.027739</td>\n",
       "      <td>-0.002885</td>\n",
       "      <td>0.001373</td>\n",
       "      <td>-0.040582</td>\n",
       "      <td>-0.004657</td>\n",
       "      <td>0.026895</td>\n",
       "      <td>0.052670</td>\n",
       "      <td>-0.035369</td>\n",
       "      <td>0.005745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.027575</td>\n",
       "      <td>0.065745</td>\n",
       "      <td>-0.046614</td>\n",
       "      <td>-0.042876</td>\n",
       "      <td>-0.006606</td>\n",
       "      <td>-0.022570</td>\n",
       "      <td>-0.026257</td>\n",
       "      <td>0.051032</td>\n",
       "      <td>-0.028213</td>\n",
       "      <td>0.030517</td>\n",
       "      <td>...</td>\n",
       "      <td>0.041123</td>\n",
       "      <td>-0.010204</td>\n",
       "      <td>-0.030535</td>\n",
       "      <td>-0.020685</td>\n",
       "      <td>0.019016</td>\n",
       "      <td>-0.022978</td>\n",
       "      <td>0.052545</td>\n",
       "      <td>-0.002048</td>\n",
       "      <td>-0.041150</td>\n",
       "      <td>-0.066599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.011732</td>\n",
       "      <td>0.020154</td>\n",
       "      <td>-0.052613</td>\n",
       "      <td>0.025233</td>\n",
       "      <td>-0.056231</td>\n",
       "      <td>-0.056353</td>\n",
       "      <td>0.091451</td>\n",
       "      <td>-0.017706</td>\n",
       "      <td>-0.005750</td>\n",
       "      <td>0.044669</td>\n",
       "      <td>...</td>\n",
       "      <td>0.058850</td>\n",
       "      <td>-0.002576</td>\n",
       "      <td>-0.043297</td>\n",
       "      <td>-0.069057</td>\n",
       "      <td>-0.025404</td>\n",
       "      <td>-0.071836</td>\n",
       "      <td>0.048371</td>\n",
       "      <td>0.016060</td>\n",
       "      <td>-0.055275</td>\n",
       "      <td>-0.078401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.015800</td>\n",
       "      <td>0.015869</td>\n",
       "      <td>-0.042117</td>\n",
       "      <td>-0.056513</td>\n",
       "      <td>0.059205</td>\n",
       "      <td>-0.003981</td>\n",
       "      <td>0.005216</td>\n",
       "      <td>0.066134</td>\n",
       "      <td>0.032295</td>\n",
       "      <td>0.068357</td>\n",
       "      <td>...</td>\n",
       "      <td>0.025379</td>\n",
       "      <td>0.054186</td>\n",
       "      <td>-0.069469</td>\n",
       "      <td>-0.017845</td>\n",
       "      <td>-0.021630</td>\n",
       "      <td>-0.029128</td>\n",
       "      <td>0.047190</td>\n",
       "      <td>-0.039558</td>\n",
       "      <td>-0.042111</td>\n",
       "      <td>-0.001511</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 512 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0 -0.004167  0.003991 -0.063261  0.051590 -0.041000 -0.044420  0.084289   \n",
       "1  0.028851 -0.046769 -0.003425  0.002011  0.038640  0.033116  0.003396   \n",
       "2 -0.027575  0.065745 -0.046614 -0.042876 -0.006606 -0.022570 -0.026257   \n",
       "3  0.011732  0.020154 -0.052613  0.025233 -0.056231 -0.056353  0.091451   \n",
       "4  0.015800  0.015869 -0.042117 -0.056513  0.059205 -0.003981  0.005216   \n",
       "\n",
       "          7         8         9    ...          502       503       504  \\\n",
       "0 -0.007363 -0.021403 -0.022497    ...     0.060429 -0.062331 -0.017937   \n",
       "1  0.041565 -0.004693  0.022073    ...     0.045903  0.027739 -0.002885   \n",
       "2  0.051032 -0.028213  0.030517    ...     0.041123 -0.010204 -0.030535   \n",
       "3 -0.017706 -0.005750  0.044669    ...     0.058850 -0.002576 -0.043297   \n",
       "4  0.066134  0.032295  0.068357    ...     0.025379  0.054186 -0.069469   \n",
       "\n",
       "        505       506       507       508       509       510       511  \n",
       "0 -0.033932  0.009563 -0.063174  0.053774 -0.043223 -0.044161 -0.066338  \n",
       "1  0.001373 -0.040582 -0.004657  0.026895  0.052670 -0.035369  0.005745  \n",
       "2 -0.020685  0.019016 -0.022978  0.052545 -0.002048 -0.041150 -0.066599  \n",
       "3 -0.069057 -0.025404 -0.071836  0.048371  0.016060 -0.055275 -0.078401  \n",
       "4 -0.017845 -0.021630 -0.029128  0.047190 -0.039558 -0.042111 -0.001511  \n",
       "\n",
       "[5 rows x 512 columns]"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>502</th>\n",
       "      <th>503</th>\n",
       "      <th>504</th>\n",
       "      <th>505</th>\n",
       "      <th>506</th>\n",
       "      <th>507</th>\n",
       "      <th>508</th>\n",
       "      <th>509</th>\n",
       "      <th>510</th>\n",
       "      <th>511</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>0.041789</td>\n",
       "      <td>0.016385</td>\n",
       "      <td>0.005377</td>\n",
       "      <td>-0.040053</td>\n",
       "      <td>0.007856</td>\n",
       "      <td>-0.051096</td>\n",
       "      <td>0.012619</td>\n",
       "      <td>0.010146</td>\n",
       "      <td>0.022841</td>\n",
       "      <td>0.054151</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000395</td>\n",
       "      <td>0.029510</td>\n",
       "      <td>0.003630</td>\n",
       "      <td>-0.011744</td>\n",
       "      <td>0.024925</td>\n",
       "      <td>0.013952</td>\n",
       "      <td>0.013481</td>\n",
       "      <td>-0.043311</td>\n",
       "      <td>-0.023502</td>\n",
       "      <td>0.042640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>0.006894</td>\n",
       "      <td>0.059347</td>\n",
       "      <td>-0.033787</td>\n",
       "      <td>0.014413</td>\n",
       "      <td>0.080144</td>\n",
       "      <td>0.026640</td>\n",
       "      <td>0.007397</td>\n",
       "      <td>0.034594</td>\n",
       "      <td>0.024090</td>\n",
       "      <td>0.059268</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003984</td>\n",
       "      <td>-0.047112</td>\n",
       "      <td>-0.011005</td>\n",
       "      <td>-0.024385</td>\n",
       "      <td>-0.008388</td>\n",
       "      <td>-0.045448</td>\n",
       "      <td>0.033893</td>\n",
       "      <td>0.009863</td>\n",
       "      <td>-0.022424</td>\n",
       "      <td>-0.033382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>0.019440</td>\n",
       "      <td>0.010227</td>\n",
       "      <td>-0.020524</td>\n",
       "      <td>0.070248</td>\n",
       "      <td>0.029649</td>\n",
       "      <td>0.011538</td>\n",
       "      <td>0.025946</td>\n",
       "      <td>0.002944</td>\n",
       "      <td>-0.068869</td>\n",
       "      <td>0.063171</td>\n",
       "      <td>...</td>\n",
       "      <td>0.056127</td>\n",
       "      <td>0.054568</td>\n",
       "      <td>0.086540</td>\n",
       "      <td>-0.078324</td>\n",
       "      <td>-0.074529</td>\n",
       "      <td>-0.002030</td>\n",
       "      <td>0.066389</td>\n",
       "      <td>0.019589</td>\n",
       "      <td>-0.039211</td>\n",
       "      <td>-0.019209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>-0.000165</td>\n",
       "      <td>-0.017769</td>\n",
       "      <td>-0.049595</td>\n",
       "      <td>0.053551</td>\n",
       "      <td>0.056311</td>\n",
       "      <td>-0.016976</td>\n",
       "      <td>0.051693</td>\n",
       "      <td>0.033347</td>\n",
       "      <td>-0.043562</td>\n",
       "      <td>0.067062</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.037130</td>\n",
       "      <td>0.064728</td>\n",
       "      <td>0.036894</td>\n",
       "      <td>-0.013267</td>\n",
       "      <td>-0.010365</td>\n",
       "      <td>-0.040716</td>\n",
       "      <td>0.048874</td>\n",
       "      <td>-0.045565</td>\n",
       "      <td>-0.056504</td>\n",
       "      <td>-0.027916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>-0.014701</td>\n",
       "      <td>0.060965</td>\n",
       "      <td>-0.032717</td>\n",
       "      <td>0.041932</td>\n",
       "      <td>0.044252</td>\n",
       "      <td>-0.068490</td>\n",
       "      <td>0.074261</td>\n",
       "      <td>-0.009464</td>\n",
       "      <td>0.041001</td>\n",
       "      <td>0.028019</td>\n",
       "      <td>...</td>\n",
       "      <td>0.053696</td>\n",
       "      <td>0.045869</td>\n",
       "      <td>0.077155</td>\n",
       "      <td>-0.012522</td>\n",
       "      <td>-0.011834</td>\n",
       "      <td>-0.004824</td>\n",
       "      <td>0.027079</td>\n",
       "      <td>-0.063949</td>\n",
       "      <td>-0.034393</td>\n",
       "      <td>-0.028559</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 512 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             0         1         2         3         4         5         6  \\\n",
       "9995  0.041789  0.016385  0.005377 -0.040053  0.007856 -0.051096  0.012619   \n",
       "9996  0.006894  0.059347 -0.033787  0.014413  0.080144  0.026640  0.007397   \n",
       "9997  0.019440  0.010227 -0.020524  0.070248  0.029649  0.011538  0.025946   \n",
       "9998 -0.000165 -0.017769 -0.049595  0.053551  0.056311 -0.016976  0.051693   \n",
       "9999 -0.014701  0.060965 -0.032717  0.041932  0.044252 -0.068490  0.074261   \n",
       "\n",
       "             7         8         9    ...          502       503       504  \\\n",
       "9995  0.010146  0.022841  0.054151    ...    -0.000395  0.029510  0.003630   \n",
       "9996  0.034594  0.024090  0.059268    ...     0.003984 -0.047112 -0.011005   \n",
       "9997  0.002944 -0.068869  0.063171    ...     0.056127  0.054568  0.086540   \n",
       "9998  0.033347 -0.043562  0.067062    ...    -0.037130  0.064728  0.036894   \n",
       "9999 -0.009464  0.041001  0.028019    ...     0.053696  0.045869  0.077155   \n",
       "\n",
       "           505       506       507       508       509       510       511  \n",
       "9995 -0.011744  0.024925  0.013952  0.013481 -0.043311 -0.023502  0.042640  \n",
       "9996 -0.024385 -0.008388 -0.045448  0.033893  0.009863 -0.022424 -0.033382  \n",
       "9997 -0.078324 -0.074529 -0.002030  0.066389  0.019589 -0.039211 -0.019209  \n",
       "9998 -0.013267 -0.010365 -0.040716  0.048874 -0.045565 -0.056504 -0.027916  \n",
       "9999 -0.012522 -0.011834 -0.004824  0.027079 -0.063949 -0.034393 -0.028559  \n",
       "\n",
       "[5 rows x 512 columns]"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 512)"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataM = data.values\n",
    "np.shape(dataM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>question_text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00002165364db923c7e6</td>\n",
       "      <td>How did Quebec nationalists see their province...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000032939017120e6e44</td>\n",
       "      <td>Do you have an adopted dog, how would you enco...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0000412ca6e4628ce2cf</td>\n",
       "      <td>Why does velocity affect time? Does velocity a...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000042bf85aa498cd78e</td>\n",
       "      <td>How did Otto von Guericke used the Magdeburg h...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0000455dfa3e01eae3af</td>\n",
       "      <td>Can I convert montra helicon D to a mountain b...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    qid                                      question_text  \\\n",
       "0  00002165364db923c7e6  How did Quebec nationalists see their province...   \n",
       "1  000032939017120e6e44  Do you have an adopted dog, how would you enco...   \n",
       "2  0000412ca6e4628ce2cf  Why does velocity affect time? Does velocity a...   \n",
       "3  000042bf85aa498cd78e  How did Otto von Guericke used the Magdeburg h...   \n",
       "4  0000455dfa3e01eae3af  Can I convert montra helicon D to a mountain b...   \n",
       "\n",
       "   target  \n",
       "0       0  \n",
       "1       0  \n",
       "2       0  \n",
       "3       0  \n",
       "4       0  "
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = pd.read_csv(\"data/train.csv\")\n",
    "labelsM = labels.values\n",
    "labels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.shape(labelsM)\n",
    "labelsM=labelsM[:10000,2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_145 (Dense)            (None, 512)               262656    \n",
      "_________________________________________________________________\n",
      "dense_146 (Dense)            (None, 1024)              525312    \n",
      "_________________________________________________________________\n",
      "dense_147 (Dense)            (None, 256)               262400    \n",
      "_________________________________________________________________\n",
      "dense_148 (Dense)            (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 1,050,625\n",
      "Trainable params: 1,050,625\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 6000 samples, validate on 4000 samples\n",
      "Epoch 1/10\n",
      "6000/6000 [==============================] - 3s 433us/step - loss: 0.1610 - acc: 0.9385 - val_loss: 0.1248 - val_acc: 0.9480\n",
      "Epoch 2/10\n",
      "6000/6000 [==============================] - 1s 238us/step - loss: 0.1141 - acc: 0.9542 - val_loss: 0.1222 - val_acc: 0.9500\n",
      "Epoch 3/10\n",
      "6000/6000 [==============================] - 1s 240us/step - loss: 0.0927 - acc: 0.9593 - val_loss: 0.1413 - val_acc: 0.9485\n",
      "Epoch 4/10\n",
      "6000/6000 [==============================] - 1s 238us/step - loss: 0.0693 - acc: 0.9703 - val_loss: 0.1534 - val_acc: 0.9463\n",
      "Epoch 5/10\n",
      "6000/6000 [==============================] - 1s 240us/step - loss: 0.0525 - acc: 0.9782 - val_loss: 0.1695 - val_acc: 0.9413\n",
      "Epoch 6/10\n",
      "6000/6000 [==============================] - 1s 241us/step - loss: 0.0389 - acc: 0.9840 - val_loss: 0.2345 - val_acc: 0.9467\n",
      "Epoch 7/10\n",
      "6000/6000 [==============================] - 1s 245us/step - loss: 0.0302 - acc: 0.9893 - val_loss: 0.2158 - val_acc: 0.9470\n",
      "Epoch 8/10\n",
      "6000/6000 [==============================] - 1s 245us/step - loss: 0.0235 - acc: 0.9920 - val_loss: 0.2345 - val_acc: 0.9365\n",
      "Epoch 9/10\n",
      "6000/6000 [==============================] - 2s 263us/step - loss: 0.0171 - acc: 0.9943 - val_loss: 0.2643 - val_acc: 0.9420\n",
      "Epoch 10/10\n",
      "6000/6000 [==============================] - 2s 252us/step - loss: 0.0162 - acc: 0.9960 - val_loss: 0.2329 - val_acc: 0.9437\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0xc0d670b38>"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  keras sequential garbage\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(512, activation='relu', input_dim=512))\n",
    "model.add(Dense(1024, activation='relu'))\n",
    "model.add(Dense(256, activation='relu'))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy', metrics = ['accuracy'])\n",
    "\n",
    "\n",
    "\n",
    "model.summary()\n",
    "\n",
    "# Train the model, iterating on the data in batches of 32 samples\n",
    "model.fit(dataM, labelsM, validation_split=0.4, shuffle=True, epochs=10, batch_size=32)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = (model.predict(dataM) >= 0.5)[:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "def precision(y_true, y_pred):\n",
    "    true_positives = np.sum(y_true * y_pred)\n",
    "    predicted_positives = np.sum(y_pred)\n",
    "    precision = true_positives / predicted_positives\n",
    "    return precision\n",
    "\n",
    "\n",
    "def recall(y_true, y_pred):\n",
    "    true_positives = np.sum(y_true * y_pred)\n",
    "    possible_positives = np.sum(y_true)\n",
    "    recall = true_positives / possible_positives\n",
    "    return recall\n",
    "\n",
    "\n",
    "def fmeasure(y_true, y_pred):\n",
    "    return 2 * precision(y_true, y_pred) * recall(y_true, y_pred)/ (precision(y_true, y_pred) + recall(y_true, y_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8353140916808149\n",
      "0.7723704866562009\n",
      "0.8026101141924958\n"
     ]
    }
   ],
   "source": [
    "print(precision(labelsM, predictions))\n",
    "print(recall(labelsM, predictions))\n",
    "print(fmeasure(labelsM, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"model.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "positional argument follows keyword argument (<ipython-input-172-d663b082e3a6>, line 37)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-172-d663b082e3a6>\"\u001b[0;36m, line \u001b[0;32m37\u001b[0m\n\u001b[0;31m    model.add(Dense(512, activation='relu', input_dim=512, kernel_regularizer.l2(reg)))\u001b[0m\n\u001b[0m                                                          ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m positional argument follows keyword argument\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import keras\n",
    "import os\n",
    "import numpy as np\n",
    "\n",
    "reg = 0.01\n",
    "\n",
    "\n",
    "embeddingsR = pd.read_csv(\"data/question_embeddings0.csv\")\n",
    "embeddings = embeddingsR.values\n",
    "\n",
    "labelsR = pd.read_csv(\"data/train.csv\")\n",
    "labels = labelsR.values[:,2]\n",
    "\n",
    "examples = 5\n",
    "\n",
    "embeddings = embeddings[:examples,:]\n",
    "labels = labels[:examples]\n",
    "\n",
    "combined = np.concatenate((embeddings, labels), axis=1)\n",
    "np.random.shuffle(combined)\n",
    "\n",
    "div1 = round(examples * 0.6)\n",
    "div2 = round(examples * 0.8)\n",
    "\n",
    "embeddings_train, labels_train = combined[:div1, :512], combined[:div1, 512]\n",
    "embeddings_crossval, labels_crossval = combined[div1:div2, :512], combined[div1:div2, 512]\n",
    "embeddings_test, labels_test = combined[div2:, :512], combined[div2:, 512]\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation\n",
    "from keras import regularizers\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(512, activation='relu', input_dim=512, kernel_regularizer.l2(reg)))\n",
    "model.add(Dense(1024, activation='relu', kernel_regularizer.l2(reg)))\n",
    "model.add(Dense(256, activation='relu',kernel_regularizer.l2(reg)))\n",
    "model.add(Dense(1, activation='sigmoid', kernel_regularizer.l2(reg)))\n",
    "model.compile(optimizer='adam', loss='binary_crossentropy', metrics = ['accuracy'])\n",
    "\n",
    "\n",
    "\n",
    "model.summary()\n",
    "\n",
    "#callback functions\n",
    "filepath=\"weights-improvement-{epoch:02d}-{val_acc:.2f}.h5\"\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "callbacks_list = [checkpoint]\n",
    "\n",
    "# Train the model, iterating on the data in batches of 32 samples\n",
    "model.fit(embeddings_train, labels_train, validation_data=(embeddings_crossval, labels_crossval), epochs=10, batch_size=32, callbacks=callbacks_list, verbose=1)\n",
    "\n",
    "model.save(\"final.h5\")\n",
    "\n",
    "#metrics definitions\n",
    "def precision(y_true, y_pred):\n",
    "    true_positives = np.sum(y_true * y_pred)\n",
    "    predicted_positives = np.sum(y_pred)\n",
    "    precision = true_positives / predicted_positives\n",
    "    return precision\n",
    "\n",
    "\n",
    "def recall(y_true, y_pred):\n",
    "    true_positives = np.sum(y_true * y_pred)\n",
    "    possible_positives = np.sum(y_true)\n",
    "    recall = true_positives / possible_positives\n",
    "    return recall\n",
    "\n",
    "\n",
    "def fmeasure(y_true, y_pred):\n",
    "    return 2 * precision(y_true, y_pred) * recall(y_true, y_pred)/ (precision(y_true, y_pred) + recall(y_true, y_pred))\n",
    "\n",
    "#crossvalidation set\n",
    "print(\"validation: \")\n",
    "predictions = (model.predict(embeddings_crossval) >= 0.5)[:,0]\n",
    "print(\"precision: \")\n",
    "print(precision(labels_crossval, predictions))\n",
    "print(\"recall: \")\n",
    "print(recall(labels_crossval, predictions))\n",
    "print(\"f1 score: \")\n",
    "print(fmeasure(labels_crossval, predictions))\n",
    "\n",
    "#test set\n",
    "print(\"test set: \")\n",
    "predictions = (model.predict(embeddings_test) >= 0.5)[:,0]\n",
    "print(\"precision: \")\n",
    "print(precision(labels_test, predictions))\n",
    "print(\"recall: \")\n",
    "print(recall(labels_test, predictions))\n",
    "print(\"f1 score: \")\n",
    "print(fmeasure(labels_test, predictions))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:quora]",
   "language": "python",
   "name": "conda-env-quora-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}