LSTM_Model1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "colab": {},
    "colab_type": "code",
    "id": "vYadQfkXkuB5",
    "outputId": "21eaaa0f-0b64-47f7-a95d-4d95c9ee18af"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/donors-choose/preprocessed_data.csv\n",
      "/kaggle/input/glove6b100dtxt/glove.6B.100d.txt\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# Any results you write to the current directory are saved as output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
    "colab": {},
    "colab_type": "code",
    "id": "O6sC7S7mkuCH"
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1FeglN1-kuCS",
    "outputId": "2f60e58c-9c48-42c8-b0b4-fe69aa1a5bb4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "embeddings_index = dict()\n",
    "f = open('/kaggle/input/glove6b100dtxt/glove.6B.100d.txt',encoding=\"utf8\")\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:])\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "print('Loaded %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JFAZwvDUkuCb"
   },
   "outputs": [],
   "source": [
    "data=pd.read_csv('/kaggle/input/donors-choose/preprocessed_data.csv',nrows=50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RhV29xnskuCl",
    "outputId": "0293c998-7af6-46ca-cabc-22774015a9bf"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 9)"
      ]
     },
     "execution_count": 5,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ttl9mCZskuCt",
    "outputId": "6a961b9f-0dd3-4665-d230-8c609c8a6584"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['school_state', 'teacher_prefix', 'project_grade_category',\n",
       "       'teacher_number_of_previously_posted_projects', 'project_is_approved',\n",
       "       'clean_categories', 'clean_subcategories', 'essay', 'price'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "o3i4T_0hkuCz"
   },
   "outputs": [],
   "source": [
    "y=data['project_is_approved']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zjTrTXiFkuC5"
   },
   "outputs": [],
   "source": [
    "data=data.drop(['project_is_approved'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZzV6zxPhkuDC"
   },
   "outputs": [],
   "source": [
    "preproc=[]\n",
    "for row in data['project_grade_category']:\n",
    "    row=row.replace('grades','')\n",
    "    row=row.replace('_prek_2','prek2')\n",
    "    row=row.replace('_3_5','3to5')\n",
    "    row=row.replace('_6_8','6to8')\n",
    "    row=row.replace('_9_12','9to12')\n",
    "    preproc.append(row)\n",
    "    \n",
    "data['project_grade_category']=preproc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nuYDGk1BkuDO"
   },
   "outputs": [],
   "source": [
    "preproc=[]\n",
    "for row in data['clean_categories']:\n",
    "    row=row.replace(' ','')\n",
    "    row=row.replace('_','')\n",
    "    preproc.append(row)\n",
    "    \n",
    "data['clean_categories']=preproc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nbLka2cJkuDX"
   },
   "outputs": [],
   "source": [
    "preproc=[]\n",
    "for row in data['clean_subcategories']:\n",
    "    row=row.replace(' ','')\n",
    "    row=row.replace('_','')\n",
    "    preproc.append(row)\n",
    "    \n",
    "data['clean_subcategories']=preproc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WvFAq_3IkuDg"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hY00SMMqkuDl"
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(data, y, test_size=0.33, stratify=y)\n",
    "X_train, X_cv, y_train, y_cv = train_test_split(X_train, y_train, test_size=0.33, stratify=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "m-wo3BNwkuDq",
    "outputId": "ce9268ac-c48a-4ba1-85f5-b3da4167b9b0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['school_state', 'teacher_prefix', 'project_grade_category',\n",
       "       'teacher_number_of_previously_posted_projects', 'clean_categories',\n",
       "       'clean_subcategories', 'essay', 'price'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 14,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "W48slF1fkuDv",
    "outputId": "8341250e-f9ea-4124-cdcf-992ab6a71e99"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential,Model\n",
    "from keras.layers import Dense,BatchNormalization\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.layers import LSTM,Input,Flatten\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.preprocessing import sequence\n",
    "from keras.optimizers import Adam\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers import BatchNormalization\n",
    "from keras.initializers import he_normal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4wuV-X1akuD1"
   },
   "source": [
    "# Model 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1kQ7MT54kuD2"
   },
   "source": [
    "### Essay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KGLTtM1gkuD3"
   },
   "outputs": [],
   "source": [
    "#credit to https://machinelearningmastery.com/use-word-embedding-layers-deep-learning-keras/\n",
    "token = Tokenizer()\n",
    "token.fit_on_texts(X_train['essay'])\n",
    "vocabulary_size = len(token.word_index)+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "b3eB4H5TkuD8"
   },
   "outputs": [],
   "source": [
    "embedding_matrix = np.zeros((vocabulary_size, 100))\n",
    "for word, i in token.word_index.items():\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0TVp3FFYkuEB"
   },
   "outputs": [],
   "source": [
    "word_count = []\n",
    "\n",
    "for e in data[\"essay\"] :\n",
    "    c = len(e.split())\n",
    "    word_count.append(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QF6j3skvkuEH"
   },
   "outputs": [],
   "source": [
    "max_word_count=max(word_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OQ8c5VeikuEM",
    "outputId": "3c07047f-6efd-49de-d2c3-93355b7009c9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "339"
      ]
     },
     "execution_count": 20,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_word_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "s0SGZI0MkuER"
   },
   "outputs": [],
   "source": [
    "train_essay=token.texts_to_sequences(X_train['essay'])\n",
    "test_essay=token.texts_to_sequences(X_test['essay'])\n",
    "cv_essay=token.texts_to_sequences(X_cv['essay'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "txEySF6rkuEY"
   },
   "outputs": [],
   "source": [
    "train_essay=pad_sequences(train_essay,maxlen=350)\n",
    "test_essay=pad_sequences(test_essay,maxlen=350)\n",
    "cv_essay=pad_sequences(cv_essay,maxlen=350)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5-1QpR6AkuEb",
    "outputId": "b7e22d89-0725-4560-bcf9-7ee3e0790107"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30527, 100)"
      ]
     },
     "execution_count": 23,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "j8lXS3Z6kuEh"
   },
   "outputs": [],
   "source": [
    "max_length=350\n",
    "input_essay = Input(shape=(max_length,))\n",
    "embedding = Embedding(vocabulary_size, 100,weights=[embedding_matrix], input_length=max_length,trainable=False)(input_essay)\n",
    "lstm=LSTM(100,return_sequences=True)(embedding)\n",
    "flatten=Flatten()(lstm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Gk4s_W-wkuEk"
   },
   "source": [
    "### School State"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zxFf05qykuEl"
   },
   "outputs": [],
   "source": [
    "enc=LabelEncoder()\n",
    "train_state=enc.fit_transform(X_train['school_state'])\n",
    "test_state=enc.transform(X_test['school_state'])\n",
    "cv_state=enc.transform(X_cv['school_state'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oWvGBJVrkuEq"
   },
   "outputs": [],
   "source": [
    "vocabulary_size  = X_train[\"school_state\"].nunique()\n",
    "embedding_size_state= int(min(np.ceil((vocabulary_size)/2), 50 ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "arO9_e81kuEt"
   },
   "outputs": [],
   "source": [
    "input_state = Input(shape=(1,))\n",
    "embedding=Embedding(vocabulary_size,embedding_size_state, input_length=1,trainable=False)(input_state)\n",
    "flatten_1=Flatten()(embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "R_ob1mjpkuEx"
   },
   "source": [
    "### Project Grade Category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UvaE5Q63kuEy"
   },
   "outputs": [],
   "source": [
    "enc=LabelEncoder()\n",
    "train_grade=enc.fit_transform(X_train['project_grade_category'])\n",
    "test_grade=enc.transform(X_test['project_grade_category'])\n",
    "cv_grade=enc.transform(X_cv['project_grade_category'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wjTOrajikuE3"
   },
   "outputs": [],
   "source": [
    "vocabulary_size  = X_train[\"project_grade_category\"].nunique()\n",
    "embedding_size_grade= int(min(np.ceil((vocabulary_size)/2), 50 ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RAMA6UQvkuE7"
   },
   "outputs": [],
   "source": [
    "input_grade = Input(shape=(1,))\n",
    "embedding=Embedding(vocabulary_size,embedding_size_grade, input_length=1,trainable=False)(input_grade)\n",
    "flatten_2=Flatten()(embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3yo2cFttkuE-"
   },
   "source": [
    "### Clean Categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "l7caEHKOkuFA"
   },
   "outputs": [],
   "source": [
    "enc=LabelEncoder()\n",
    "train_cat=enc.fit_transform(X_train['clean_categories'])\n",
    "test_cat=enc.fit_transform(X_test['clean_categories'])\n",
    "cv_cat=enc.fit_transform(X_cv['clean_categories'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "d1DnKTU5kuFD"
   },
   "outputs": [],
   "source": [
    "vocabulary_size  = X_train[\"clean_categories\"].nunique()\n",
    "embedding_size_cat= int(min(np.ceil((vocabulary_size)/2), 50 ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TwnY2hl0kuFG"
   },
   "outputs": [],
   "source": [
    "input_cat = Input(shape=(1,))\n",
    "embedding=Embedding(vocabulary_size,embedding_size_cat, input_length=1,trainable=False)(input_cat)\n",
    "flatten_3=Flatten()(embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "oI8tnREBkuFJ"
   },
   "source": [
    "### Clean Subcategories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_3s1uzy7kuFK"
   },
   "outputs": [],
   "source": [
    "enc=LabelEncoder()\n",
    "train_sub=enc.fit_transform(X_train['clean_subcategories'])\n",
    "test_sub=enc.fit_transform(X_test['clean_subcategories'])\n",
    "cv_sub=enc.fit_transform(X_cv['clean_subcategories'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kscxiZVrkuFN"
   },
   "outputs": [],
   "source": [
    "vocabulary_size  = X_train[\"clean_subcategories\"].nunique()\n",
    "embedding_size_sub= int(min(np.ceil((vocabulary_size)/2), 50 ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kdtsOE_0kuFQ"
   },
   "outputs": [],
   "source": [
    "input_sub = Input(shape=(1,))\n",
    "embedding=Embedding(vocabulary_size,embedding_size_sub, input_length=1,trainable=False)(input_sub)\n",
    "flatten_4=Flatten()(embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "CFSjFr7qkuFZ"
   },
   "source": [
    "### Teacher Prefix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oKBA45ROkuFb"
   },
   "outputs": [],
   "source": [
    "enc=LabelEncoder()\n",
    "train_prefix=enc.fit_transform(X_train['teacher_prefix'])\n",
    "test_prefix=enc.transform(X_test['teacher_prefix'])\n",
    "cv_prefix=enc.transform(X_cv['teacher_prefix'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DGr0ygO1kuFd"
   },
   "outputs": [],
   "source": [
    "vocabulary_size  = X_train[\"teacher_prefix\"].nunique()\n",
    "embedding_size_prefix= int(min(np.ceil((vocabulary_size)/2), 50 ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "emNY1muMkuFg"
   },
   "outputs": [],
   "source": [
    "input_prefix = Input(shape=(1,))\n",
    "embedding=Embedding(vocabulary_size, embedding_size_prefix, input_length=1,trainable=False)(input_prefix)\n",
    "flatten_5=Flatten()(embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "JEj3dCg1kuFl"
   },
   "source": [
    "### Remaining Numerical Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "M_dzIOfQkuFn"
   },
   "outputs": [],
   "source": [
    "from scipy.sparse import hstack\n",
    "\n",
    "train_price=X_train['price'].values.reshape(-1, 1)\n",
    "train_number=X_train['teacher_number_of_previously_posted_projects'].values.reshape(-1, 1)\n",
    "\n",
    "test_price=X_test['price'].values.reshape(-1, 1)\n",
    "test_number=X_test['teacher_number_of_previously_posted_projects'].values.reshape(-1, 1)\n",
    "\n",
    "cv_price=X_cv['price'].values.reshape(-1, 1)\n",
    "cv_price=X_cv['teacher_number_of_previously_posted_projects'].values.reshape(-1, 1)\n",
    "\n",
    "rem_train=np.concatenate((train_price,train_number),axis=1)\n",
    "rem_test=np.concatenate((test_price,test_number),axis=1)\n",
    "rem_cv=np.concatenate((cv_price,cv_price),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QXDRcQhWkuFr"
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler=MinMaxScaler()\n",
    "scaler.fit(rem_train)\n",
    "rem_train_standard=scaler.transform(rem_train)\n",
    "rem_test_standard=scaler.transform(rem_test)\n",
    "rem_cv_standard=scaler.transform(rem_cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "U7AS8nofkuFu"
   },
   "outputs": [],
   "source": [
    "rem_feat = Input(shape=(2,))\n",
    "rem_feat_dense= Dense(128, activation='relu')(rem_feat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "OoDO2KdDkuFz"
   },
   "source": [
    "### Concatenating Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LpZQT0fJkuFz"
   },
   "outputs": [],
   "source": [
    "train_data=[train_essay,np.array(train_state),np.array(train_grade),np.array(train_cat),np.array(train_sub),np.array(train_prefix),np.array(rem_train_standard)]\n",
    "test_data=[test_essay,np.array(test_state),np.array(test_grade),np.array(test_cat),np.array(test_sub),np.array(test_prefix),np.array(rem_test_standard)]\n",
    "cv_data=[cv_essay,np.array(cv_state),np.array(cv_grade),np.array(cv_cat),np.array(cv_sub),np.array(cv_prefix),np.array(rem_cv_standard)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hgCqi8-UkuF1"
   },
   "source": [
    "### Concatenating all the Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QGvIrzjrkuF2"
   },
   "outputs": [],
   "source": [
    "from keras.layers import concatenate\n",
    "from keras.layers import Dropout\n",
    "concat = concatenate([flatten,flatten_1,flatten_2,flatten_3,flatten_4,flatten_5,rem_feat_dense])\n",
    "\n",
    "x = Dense(128, activation='relu',kernel_initializer=he_normal())(concat)\n",
    "x=Dropout(0.6)(x)\n",
    "\n",
    "x = Dense(256, activation='relu',kernel_initializer=he_normal())(x)\n",
    "x=Dropout(0.6)(x)\n",
    "x=BatchNormalization()(x)\n",
    "\n",
    "x = Dense(64, activation='relu',kernel_initializer=he_normal())(x)\n",
    "\n",
    "# And finally we add the main logistic regression layer\n",
    "main_output = Dense(1, activation='sigmoid',kernel_initializer=he_normal())(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JKciutVCkuF4"
   },
   "outputs": [],
   "source": [
    "model1 = Model(inputs=[input_essay, input_state, input_grade, input_cat, input_sub, input_prefix,rem_feat], outputs=[main_output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XixEm_C5kuF8",
    "outputId": "38b76a33-2353-4dcc-f36f-c4d89c9a6eee"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 350)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 350, 100)     3052700     input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_3 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_4 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_5 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_6 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   (None, 350, 100)     80400       embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 1, 26)        1326        input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_3 (Embedding)         (None, 1, 2)         8           input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_4 (Embedding)         (None, 1, 23)        1035        input_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_5 (Embedding)         (None, 1, 50)        16150       input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_6 (Embedding)         (None, 1, 3)         15          input_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "input_7 (InputLayer)            (None, 2)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 35000)        0           lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)             (None, 26)           0           embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_3 (Flatten)             (None, 2)            0           embedding_3[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_4 (Flatten)             (None, 23)           0           embedding_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_5 (Flatten)             (None, 50)           0           embedding_5[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_6 (Flatten)             (None, 3)            0           embedding_6[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 128)          384         input_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 35232)        0           flatten_1[0][0]                  \n",
      "                                                                 flatten_2[0][0]                  \n",
      "                                                                 flatten_3[0][0]                  \n",
      "                                                                 flatten_4[0][0]                  \n",
      "                                                                 flatten_5[0][0]                  \n",
      "                                                                 flatten_6[0][0]                  \n",
      "                                                                 dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 128)          4509824     concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 128)          0           dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 256)          33024       dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 256)          0           dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 256)          1024        dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 64)           16448       batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 1)            65          dense_4[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 7,712,403\n",
      "Trainable params: 4,640,657\n",
      "Non-trainable params: 3,071,746\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IEIT5Y4dkuF_"
   },
   "outputs": [],
   "source": [
    "# https://www.youtube.com/watch?v=2U6Jl7oqRkM\n",
    "# https://www.tensorflow.org/tensorboard/r2/get_started\n",
    "\n",
    "import tensorflow as tf\n",
    "from keras.callbacks import TensorBoard,EarlyStopping\n",
    "from time import time\n",
    "earlystop_1 = EarlyStopping(monitor = 'val_loss', mode=\"min\",min_delta = 0, patience = 5,verbose = 1,restore_best_weights = True)\n",
    "tensorboard_1 = TensorBoard(\"logs1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4-SL4zmWkuGB"
   },
   "outputs": [],
   "source": [
    "# https://github.com/pranaya-mathur/Donors-Choose-LSTM/blob/master/DonorsChoose_Model_1_13_Aug_19.ipynb\n",
    "import tensorflow as tf\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "def auc_score(y_true, y_pred):\n",
    "    if len(np.unique(y_true[:])) == 1:\n",
    "        return 0.5\n",
    "    else:\n",
    "        return roc_auc_score(y_true, y_pred)\n",
    "\n",
    "def auc_sc(y_true, y_pred):\n",
    "    return tf.py_func(auc_score, (y_true, y_pred), tf.double)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JB-MS8ewkuGE"
   },
   "outputs": [],
   "source": [
    "from keras.utils import np_utils\n",
    "Y_train=np_utils.to_categorical(y_train)\n",
    "Y_test=np_utils.to_categorical(y_test)\n",
    "Y_cv=np_utils.to_categorical(y_cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZR2dzWLnkuGG"
   },
   "outputs": [],
   "source": [
    "model1.compile(optimizer='adam', loss='binary_crossentropy', metrics=[auc_sc])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nFl69QYvkuGJ",
    "outputId": "cb2808d2-b74e-4a59-eec2-81368af7f15c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 22445 samples, validate on 11055 samples\n",
      "Epoch 1/12\n",
      "22445/22445 [==============================] - 27s 1ms/step - loss: 0.5433 - auc_sc: 0.5176 - val_loss: 0.4849 - val_auc_sc: 0.4917\n",
      "Epoch 2/12\n",
      "22445/22445 [==============================] - 25s 1ms/step - loss: 0.4514 - auc_sc: 0.5758 - val_loss: 0.4948 - val_auc_sc: 0.6761\n",
      "Epoch 3/12\n",
      "22445/22445 [==============================] - 25s 1ms/step - loss: 0.4348 - auc_sc: 0.6291 - val_loss: 0.4723 - val_auc_sc: 0.6977\n",
      "Epoch 4/12\n",
      "22445/22445 [==============================] - 25s 1ms/step - loss: 0.4236 - auc_sc: 0.6702 - val_loss: 0.5308 - val_auc_sc: 0.7056\n",
      "Epoch 5/12\n",
      "22445/22445 [==============================] - 25s 1ms/step - loss: 0.4191 - auc_sc: 0.6764 - val_loss: 0.5358 - val_auc_sc: 0.7161\n",
      "Epoch 6/12\n",
      "22445/22445 [==============================] - 26s 1ms/step - loss: 0.4135 - auc_sc: 0.6910 - val_loss: 0.4843 - val_auc_sc: 0.7188\n",
      "Epoch 7/12\n",
      "22445/22445 [==============================] - 25s 1ms/step - loss: 0.4108 - auc_sc: 0.6982 - val_loss: 0.4961 - val_auc_sc: 0.7279\n",
      "Epoch 8/12\n",
      "22445/22445 [==============================] - 25s 1ms/step - loss: 0.4052 - auc_sc: 0.7100 - val_loss: 0.4958 - val_auc_sc: 0.7270\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00008: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f4a3a860828>"
      ]
     },
     "execution_count": 51,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1.fit(train_data,y_train,batch_size=512, epochs=12,validation_data=(cv_data,y_cv),verbose=1,class_weight='balanced',callbacks=[tensorboard_1,earlystop_1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-17T13:17:07.231063Z",
     "start_time": "2019-10-17T13:17:07.216991Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+-----------+----------+\n",
      "|  Model  | train_auc | test_auc |\n",
      "+---------+-----------+----------+\n",
      "| Model_1 |    0.71   |  0.727   |\n",
      "+---------+-----------+----------+\n"
     ]
    }
   ],
   "source": [
    "from prettytable import PrettyTable\n",
    "x = PrettyTable()\n",
    "x.field_names = [\"Model\", \"train_auc\", \"test_auc\"]\n",
    "x.add_row([\"Model_1\", 0.7100,0.7270])\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "donors-choose-lstm(6).ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}