Quora-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import keras\n",
    "import os\n",
    "import numpy as np\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Data Frame Dimensions: (1306122, 3)\n"
     ]
    }
   ],
   "source": [
    "# read training data\n",
    "train_df = pd.read_csv(\"data/train.csv\")\n",
    "print(\"Training Data Frame Dimensions: \" + str(train_df.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>question_text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00002165364db923c7e6</td>\n",
       "      <td>How did Quebec nationalists see their province...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000032939017120e6e44</td>\n",
       "      <td>Do you have an adopted dog, how would you enco...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0000412ca6e4628ce2cf</td>\n",
       "      <td>Why does velocity affect time? Does velocity a...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000042bf85aa498cd78e</td>\n",
       "      <td>How did Otto von Guericke used the Magdeburg h...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0000455dfa3e01eae3af</td>\n",
       "      <td>Can I convert montra helicon D to a mountain b...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    qid                                      question_text  \\\n",
       "0  00002165364db923c7e6  How did Quebec nationalists see their province...   \n",
       "1  000032939017120e6e44  Do you have an adopted dog, how would you enco...   \n",
       "2  0000412ca6e4628ce2cf  Why does velocity affect time? Does velocity a...   \n",
       "3  000042bf85aa498cd78e  How did Otto von Guericke used the Magdeburg h...   \n",
       "4  0000455dfa3e01eae3af  Can I convert montra helicon D to a mountain b...   \n",
       "\n",
       "   target  \n",
       "0       0  \n",
       "1       0  \n",
       "2       0  \n",
       "3       0  \n",
       "4       0  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# first 5 elements of training data\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>question_text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1306117</th>\n",
       "      <td>ffffcc4e2331aaf1e41e</td>\n",
       "      <td>What other technical skills do you need as a c...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1306118</th>\n",
       "      <td>ffffd431801e5a2f4861</td>\n",
       "      <td>Does MS in ECE have good job prospects in USA ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1306119</th>\n",
       "      <td>ffffd48fb36b63db010c</td>\n",
       "      <td>Is foam insulation toxic?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1306120</th>\n",
       "      <td>ffffec519fa37cf60c78</td>\n",
       "      <td>How can one start a research project based on ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1306121</th>\n",
       "      <td>ffffed09fedb5088744a</td>\n",
       "      <td>Who wins in a battle between a Wolverine and a...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          qid  \\\n",
       "1306117  ffffcc4e2331aaf1e41e   \n",
       "1306118  ffffd431801e5a2f4861   \n",
       "1306119  ffffd48fb36b63db010c   \n",
       "1306120  ffffec519fa37cf60c78   \n",
       "1306121  ffffed09fedb5088744a   \n",
       "\n",
       "                                             question_text  target  \n",
       "1306117  What other technical skills do you need as a c...       0  \n",
       "1306118  Does MS in ECE have good job prospects in USA ...       0  \n",
       "1306119                          Is foam insulation toxic?       0  \n",
       "1306120  How can one start a research project based on ...       0  \n",
       "1306121  Who wins in a battle between a Wolverine and a...       0  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# last 5 elements of training data\n",
    "train_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Data Frame Dimensions: (1306122, 3)\n"
     ]
    }
   ],
   "source": [
    "# read test data\n",
    "test_df = pd.read_csv(\"data/train.csv\")\n",
    "print(\"Test Data Frame Dimensions: \" + str(test_df.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>question_text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00002165364db923c7e6</td>\n",
       "      <td>How did Quebec nationalists see their province...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000032939017120e6e44</td>\n",
       "      <td>Do you have an adopted dog, how would you enco...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0000412ca6e4628ce2cf</td>\n",
       "      <td>Why does velocity affect time? Does velocity a...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000042bf85aa498cd78e</td>\n",
       "      <td>How did Otto von Guericke used the Magdeburg h...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0000455dfa3e01eae3af</td>\n",
       "      <td>Can I convert montra helicon D to a mountain b...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    qid                                      question_text  \\\n",
       "0  00002165364db923c7e6  How did Quebec nationalists see their province...   \n",
       "1  000032939017120e6e44  Do you have an adopted dog, how would you enco...   \n",
       "2  0000412ca6e4628ce2cf  Why does velocity affect time? Does velocity a...   \n",
       "3  000042bf85aa498cd78e  How did Otto von Guericke used the Magdeburg h...   \n",
       "4  0000455dfa3e01eae3af  Can I convert montra helicon D to a mountain b...   \n",
       "\n",
       "   target  \n",
       "0       0  \n",
       "1       0  \n",
       "2       0  \n",
       "3       0  \n",
       "4       0  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>question_text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1306117</th>\n",
       "      <td>ffffcc4e2331aaf1e41e</td>\n",
       "      <td>What other technical skills do you need as a c...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1306118</th>\n",
       "      <td>ffffd431801e5a2f4861</td>\n",
       "      <td>Does MS in ECE have good job prospects in USA ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1306119</th>\n",
       "      <td>ffffd48fb36b63db010c</td>\n",
       "      <td>Is foam insulation toxic?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1306120</th>\n",
       "      <td>ffffec519fa37cf60c78</td>\n",
       "      <td>How can one start a research project based on ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1306121</th>\n",
       "      <td>ffffed09fedb5088744a</td>\n",
       "      <td>Who wins in a battle between a Wolverine and a...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          qid  \\\n",
       "1306117  ffffcc4e2331aaf1e41e   \n",
       "1306118  ffffd431801e5a2f4861   \n",
       "1306119  ffffd48fb36b63db010c   \n",
       "1306120  ffffec519fa37cf60c78   \n",
       "1306121  ffffed09fedb5088744a   \n",
       "\n",
       "                                             question_text  target  \n",
       "1306117  What other technical skills do you need as a c...       0  \n",
       "1306118  Does MS in ECE have good job prospects in USA ...       0  \n",
       "1306119                          Is foam insulation toxic?       0  \n",
       "1306120  How can one start a research project based on ...       0  \n",
       "1306121  Who wins in a battle between a Wolverine and a...       0  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate dummy data\n",
    "import numpy as np\n",
    "\n",
    "labels = np.random.randint(2, size=(1000, 1))\n",
    "data = np.random.random((1000, 512))\n",
    "test = data\n",
    "results = labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.85314064 0.09459427 0.77902909 ... 0.30016484 0.12092971 0.91058521]\n",
      " [0.69265633 0.91436831 0.38139414 ... 0.38265083 0.7028286  0.05930205]\n",
      " [0.80646502 0.49092206 0.2074234  ... 0.19946205 0.01187369 0.80270837]\n",
      " ...\n",
      " [0.3917083  0.41233252 0.59315065 ... 0.66575081 0.99449461 0.96517336]\n",
      " [0.22466286 0.41064903 0.97671977 ... 0.22063949 0.954963   0.03142363]\n",
      " [0.09900612 0.13169242 0.58193975 ... 0.04142291 0.4177496  0.57039056]]\n"
     ]
    }
   ],
   "source": [
    "print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_29 (Dense)             (None, 512)               262656    \n",
      "_________________________________________________________________\n",
      "dense_30 (Dense)             (None, 1024)              525312    \n",
      "_________________________________________________________________\n",
      "dense_31 (Dense)             (None, 256)               262400    \n",
      "_________________________________________________________________\n",
      "dense_32 (Dense)             (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 1,050,625\n",
      "Trainable params: 1,050,625\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/30\n",
      "1000/1000 [==============================] - 1s 519us/step - loss: 0.7271 - acc: 0.5050\n",
      "Epoch 2/30\n",
      "1000/1000 [==============================] - 0s 38us/step - loss: 1.4721 - acc: 0.4950\n",
      "Epoch 3/30\n",
      "1000/1000 [==============================] - 0s 40us/step - loss: 0.7934 - acc: 0.4950\n",
      "Epoch 4/30\n",
      "1000/1000 [==============================] - 0s 39us/step - loss: 0.7022 - acc: 0.5050\n",
      "Epoch 5/30\n",
      "1000/1000 [==============================] - 0s 40us/step - loss: 0.7343 - acc: 0.5050\n",
      "Epoch 6/30\n",
      "1000/1000 [==============================] - 0s 38us/step - loss: 0.7112 - acc: 0.5050\n",
      "Epoch 7/30\n",
      "1000/1000 [==============================] - 0s 39us/step - loss: 0.6920 - acc: 0.5090\n",
      "Epoch 8/30\n",
      "1000/1000 [==============================] - 0s 42us/step - loss: 0.6998 - acc: 0.4950\n",
      "Epoch 9/30\n",
      "1000/1000 [==============================] - 0s 40us/step - loss: 0.7001 - acc: 0.4950\n",
      "Epoch 10/30\n",
      "1000/1000 [==============================] - 0s 39us/step - loss: 0.6923 - acc: 0.4960\n",
      "Epoch 11/30\n",
      "1000/1000 [==============================] - 0s 38us/step - loss: 0.6899 - acc: 0.5130\n",
      "Epoch 12/30\n",
      "1000/1000 [==============================] - 0s 38us/step - loss: 0.6931 - acc: 0.5050\n",
      "Epoch 13/30\n",
      "1000/1000 [==============================] - 0s 39us/step - loss: 0.6920 - acc: 0.5050\n",
      "Epoch 14/30\n",
      "1000/1000 [==============================] - 0s 38us/step - loss: 0.6878 - acc: 0.5070\n",
      "Epoch 15/30\n",
      "1000/1000 [==============================] - 0s 42us/step - loss: 0.6861 - acc: 0.6570\n",
      "Epoch 16/30\n",
      "1000/1000 [==============================] - 0s 40us/step - loss: 0.6863 - acc: 0.5180\n",
      "Epoch 17/30\n",
      "1000/1000 [==============================] - 0s 39us/step - loss: 0.6844 - acc: 0.5360\n",
      "Epoch 18/30\n",
      "1000/1000 [==============================] - 0s 41us/step - loss: 0.6807 - acc: 0.6900\n",
      "Epoch 19/30\n",
      "1000/1000 [==============================] - 0s 40us/step - loss: 0.6795 - acc: 0.5320\n",
      "Epoch 20/30\n",
      "1000/1000 [==============================] - 0s 39us/step - loss: 0.6764 - acc: 0.5440\n",
      "Epoch 21/30\n",
      "1000/1000 [==============================] - 0s 41us/step - loss: 0.6714 - acc: 0.6980\n",
      "Epoch 22/30\n",
      "1000/1000 [==============================] - 0s 41us/step - loss: 0.6684 - acc: 0.6470\n",
      "Epoch 23/30\n",
      "1000/1000 [==============================] - 0s 46us/step - loss: 0.6616 - acc: 0.6960\n",
      "Epoch 24/30\n",
      "1000/1000 [==============================] - 0s 43us/step - loss: 0.6573 - acc: 0.6280\n",
      "Epoch 25/30\n",
      "1000/1000 [==============================] - 0s 42us/step - loss: 0.6482 - acc: 0.7160\n",
      "Epoch 26/30\n",
      "1000/1000 [==============================] - 0s 39us/step - loss: 0.6384 - acc: 0.7240\n",
      "Epoch 27/30\n",
      "1000/1000 [==============================] - 0s 41us/step - loss: 0.6325 - acc: 0.6750\n",
      "Epoch 28/30\n",
      "1000/1000 [==============================] - 0s 40us/step - loss: 0.6377 - acc: 0.6060\n",
      "Epoch 29/30\n",
      "1000/1000 [==============================] - 0s 40us/step - loss: 0.6298 - acc: 0.6110\n",
      "Epoch 30/30\n",
      "1000/1000 [==============================] - 0s 40us/step - loss: 0.6026 - acc: 0.7430\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0xb9414e5c0>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  keras sequential garbage\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(512, activation='relu', input_dim=512))\n",
    "model.add(Dense(1024, activation='relu'))\n",
    "model.add(Dense(256, activation='relu'))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "\n",
    "\n",
    "model.summary()\n",
    "# Train the model, iterating on the data in batches of 32 samples\n",
    "model.fit(test, results, epochs=30, batch_size=1000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "674"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum((model.predict(test)>= 0.5) == results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:quora]",
   "language": "python",
   "name": "conda-env-quora-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}