imbalanced learn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "training = pd.read_csv('trainingVect.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training['Ow2'] = 0\n",
    "training.ix[training['word2'].str[-1] == 'O','Ow2'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "heldout = pd.read_csv('heldoutVect.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dvan = heldout[['word1','word2','tag','label']][heldout.word2.str[-1]== 'O'].groupby('label').count()\n",
    "dvan.sort_values(by='word1',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "heldout[['word1','word2','tag','label']][(heldout['word2'].str[-2:] == 'EH') & (heldout['label']==2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "heldout['Ow2'] = 0\n",
    "heldout.ix[training['word2'].str[-1] == 'O','Ow2'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "heldout.head(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ignore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "finRemove = eval(open('adap2.features').read())\n",
    "type(finRemove)\n",
    "finRemove[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for item in finRemove:\n",
    "    del heldout[item]\n",
    "    del training[item]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "heldout.columns.difference(training.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print len(heldout.columns),len(training.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,item in enumerate(training.columns):\n",
    "    if training.columns[i] != heldout.columns[i]:\n",
    "        print training.columns[i],heldout.columns[i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stop Ignorring"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print training.columns[-41:]\n",
    "\n",
    "allCols = list(training.columns[5:-39])\n",
    "allCols.insert(-1,'Ow2')\n",
    "allCols[-3:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import *\n",
    "\n",
    "\n",
    "#featVect = featVect.sample(frac=1.0)\n",
    "features = array(training.ix[:,allCols])\n",
    "labels = array(training.ix[:,'label'])\n",
    "\n",
    "heldFeatures = array(heldout.ix[:,allCols])\n",
    "\n",
    "heldLabels = array(heldout.ix[:,'label'])\n",
    "print allCols[-2:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = training.columns\n",
    "a[26:413]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import StratifiedShuffleSplit\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "skf = StratifiedKFold(labels, n_folds=10,shuffle=True)\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = RandomForestClassifier(n_jobs=-1,n_estimators=400,max_features='log2')\n",
    "clf  = clf.fit(features,labels)\n",
    "predicted_label = clf.predict(heldFeatures)\n",
    "pred_prob = clf.predict_proba(heldFeatures)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print classification_report(heldLabels, predicted_label)\n",
    "print ('Accuracy:',accuracy_score(heldLabels, predicted_label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print classification_report(heldLabels, predicted_label)\n",
    "print ('Accuracy:',accuracy_score(heldLabels, predicted_label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "forest = clf\n",
    "importances = forest.feature_importances_\n",
    "std = np.std([tree.feature_importances_ for tree in forest.estimators_],\n",
    "             axis=0)\n",
    "indices = np.argsort(importances)[::-1]\n",
    "X = features\n",
    "# Print the feature ranking\n",
    "print(\"Feature ranking:\")\n",
    "\n",
    "\"\"\"for f in range(features.shape[1]):\n",
    "    print(\"%d. feature %d (%f)\" % (f + 1, indices[f], importances[indices[f]]))\n",
    "\"\"\"\n",
    "# Plot the feature importances of the forest\n",
    "plt.figure()\n",
    "plt.title(\"Feature importances\")\n",
    "plt.bar(range(X.shape[1]), importances[indices],\n",
    "       color=\"r\", yerr=std[indices], align=\"center\")\n",
    "plt.xticks(range(X.shape[1]), indices)\n",
    "plt.xlim([-1, X.shape[1]])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "triple = zip(range(len(heldLabels)),heldLabels,predicted_label)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prob =pd.DataFrame().from_records(pred_prob,columns=['prob0','prob1','prob2','prob3'])\n",
    "prob = pd.merge(prob,result,left_index=True,right_index=True,how='inner')\n",
    "prob[(prob['heldLabels']!= prob['predicted_label']) & (prob['heldLabels'] == 2)][['prob0','prob1','prob2','prob3','word1','word2','heldLabels','predicted_label','dups','tag']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prob[(prob['word1'].str[-1] == 'a') & (prob['word2'].str[-1] == 'm') ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trip = pd.DataFrame().from_records(triple,columns=['index','heldLabels','predicted_label'])\n",
    "\n",
    "result = pd.merge(trip,heldout,how='inner',left_on='index',right_index=True,)\n",
    "result[(result['heldLabels']!= result['predicted_label']) & (result['heldLabels'] == 2)][['word1','word2','heldLabels','predicted_label','dups','tag']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extremely Randomied trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomTreesEmbedding, ExtraTreesClassifier\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from sklearn.metrics import accuracy_score\n",
    "trees = ExtraTreesClassifier(n_estimators=400, random_state=43, max_features='log2')\n",
    "trees = trees.fit(features,labels)\n",
    "\n",
    "predicted_label = trees.predict(heldFeatures)\n",
    "print classification_report(heldLabels, predicted_label)\n",
    "print ('Accuracy:',accuracy_score(heldLabels, predicted_label))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import svm\n",
    "clf = svm.SVC()\n",
    "clf.fit(features, labels)  \n",
    "predicted_label = clf.predict(heldFeatures)\n",
    "print classification_report(heldLabels, predicted_label)\n",
    "print ('Accuracy:',accuracy_score(heldLabels, predicted_label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import tree\n",
    "clf = tree.DecisionTreeClassifier()\n",
    "clf.fit(features, labels)  \n",
    "predicted_label = clf.predict(heldFeatures)\n",
    "print classification_report(heldLabels, predicted_label)\n",
    "print ('Accuracy:',accuracy_score(heldLabels, predicted_label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy\n",
    "alist = zip(list(a[5:-29]),list(trees.feature_importances_))\n",
    "feati = open('featImp.csv','w')\n",
    "for item in alist:\n",
    "    print >>feati,item[0],',',item[1]\n",
    "feati.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "clf = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0,max_depth=10).fit(features,labels)\n",
    "predicted_label = clf.predict(heldFeatures)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print classification_report(heldLabels, predicted_label)\n",
    "print ('Accuracy:',accuracy_score(heldLabels, predicted_label))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Easy Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from imblearn.ensemble import BalanceCascade\n",
    "from imblearn.ensemble import EasyEnsemble\n",
    "ee = EasyEnsemble(ratio=0.055,random_state=25,replacement=True,n_subsets=20)\n",
    "X_res, y_res = ee.fit_sample(features, labels)\n",
    "print('Resampled dataset shape {}'.format(Counter(y_res[1])))\n",
    "\n",
    "\"\"\"from sklearn.decomposition import PCA\n",
    "pcaHeld = PCA(n_components=900).fit(features)\n",
    "pcaFe = pcaHeld.transform(features)\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_res[1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "#clf = AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=100,splitter='best'),n_estimators=100,learning_rate=1.0)\n",
    "clf = ExtraTreesClassifier(n_estimators=200, max_features='log2', random_state=43, warm_start=True,n_jobs=-1)\n",
    "countrs= []\n",
    "for i in range(len(heldFeatures)):\n",
    "    countrs.append([0]*4)\n",
    "\n",
    "for i,item in enumerate(X_res):\n",
    "    print i,\n",
    "    clf.fit(X_res[i],y_res[i])\n",
    "    staged_predict = clf.predict_proba(heldFeatures)\n",
    "    clf.n_estimators += 50\n",
    "    for j,stuff in enumerate(staged_predict):\n",
    "        for k,thing in enumerate(list(stuff)):\n",
    "            countrs[j][k] += thing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "majLabels = [-1]*len(countrs)\n",
    "for i,item in enumerate(countrs):\n",
    "    majLabels[i] = item.index(max(item))\n",
    "print classification_report(heldLabels, majLabels)\n",
    "print ('Accuracy:',accuracy_score(heldLabels, majLabels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = open('preds.csv','w')\n",
    "for i,item in enumerate(heldLabels):\n",
    "    print>>preds,item,',',majLabels[i]\n",
    "preds.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print type(heldLabels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from sklearn.metrics import accuracy_score\n",
    "import pandas \n",
    "preds = pandas.read_csv('myList.csv')\n",
    "import numpy\n",
    "a = list(preds.label)\n",
    "b = list(preds.predicted)\n",
    "print classification_report(a,b)\n",
    "print ('Accuracy:',accuracy_score(a, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds[(preds['tag']=='Bsmn')].groupby('predicted').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from sklearn.metrics import accuracy_score\n",
    "import pandas \n",
    "preds = pandas.read_csv('preds2.csv')\n",
    "import numpy\n",
    "a = list(preds.orig)\n",
    "b = list(preds.pred)\n",
    "print classification_report(a,b)\n",
    "print ('Accuracy:',accuracy_score(a, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clfFull = AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=100,splitter='best'),n_estimators=600,learning_rate=1.0)\n",
    "clfFull = clfFull.fit(features,labels)\n",
    "predicted_label = clfFull.predict(heldFeatures)\n",
    "print classification_report(heldLabels, predicted_label)\n",
    "print ('Accuracy:',accuracy_score(heldLabels, predicted_label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "akie = training.select_dtypes(include=['float64']).columns\n",
    "akie = akie.insert(-1,'label')\n",
    "pkie = training[akie].groupby('label').sum().T\n",
    "df = pkie\n",
    "df['sum'] = df.sum(axis=1)\n",
    "df['entropy'] = (df.ix[:,:3].div(df['sum'],axis=0)*np.log(df.ix[:,:3].div(df['sum'],axis=0))).sum(axis=1)\n",
    "pkie.to_csv('summar1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training[training['label']==2][['word1','word2','tag','label','dups']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ap = list(training[training['tag'] == 'Ds' ]['word2'])\n",
    "ap1 = list(training[training['tag'] == 'Ds' ]['word1'])\n",
    "#vaswra\t2.6.115.2.1\tnapuM.\tmanuRyavargaH\tvaswram\t\t\t\t\t\t\t\t\t\tvaswram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kAndas = pd.read_csv('all_kANdas.csv')\n",
    "jusWord = list(kAndas['%0_Word'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainFi = [item.split(',') for item in open('trainigFiltered.csv').read().splitlines()[1:]]\n",
    "import pandas as pd\n",
    "merger = pd.merge(training,kAndas,how='left',left_on='word1',right_on='%0_Word',suffixes=('_x','_word1'))\n",
    "merger = pd.merge(merger,kAndas,how='left',left_on='word2',right_on='%0_Word',suffixes=('_x','_word2'))\n",
    "merger\n",
    "cat_columns =merger.select_dtypes(include=['object']).columns\n",
    "\n",
    "cat_columns = cat_columns[7:]\n",
    "for item in cat_columns:\n",
    "    merger[item] = merger[item].astype('category')\n",
    "\n",
    "merger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ctr = 0\n",
    "for i,item in enumerate(ap):\n",
    "    if item not in jusWord:\n",
    "        if item[:-1] not in jusWord:\n",
    "            if item[:-3] not in jusWord:\n",
    "                if item[:-1]+'a' not in jusWord:\n",
    "                    print i,item\n",
    "                    ctr += 1\n",
    "    \n",
    "print ctr,len(ap)\n",
    "\n",
    "#word1 = -ya+a, -aXi+, -pra+, -+n\n",
    "#word2 = -m+ -ena+a -e+a\n",
    "## vit, AkarRaNa, saMmAna\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = [item.split(',') for item in open('heldoutFiltered.csv').read().splitlines()[1:]]\n",
    "kAndas = [item.split(',') for item in open('all_kANdas.csv').read().splitlines()]\n",
    "nanList = ['None']*(len(kAndas[-1]) - 3)\n",
    "ctr = 0\n",
    "ctr2 = 0\n",
    "for i,item in enumerate(f):\n",
    "    matched = 0\n",
    "    matched2 = 0\n",
    "    for j,stuff in enumerate(kAndas):\n",
    "        if item[1] == stuff[0]:\n",
    "            f[i].extend(kAndas[j][2:])\n",
    "            matched= 1\n",
    "            ctr += 1\n",
    "            break\n",
    "    matched2 = 0\n",
    "    if matched == 0:\n",
    "        f[i].extend(nanList)\n",
    "\n",
    "    for j,stuff in enumerate(kAndas):\n",
    "        if item[2] == stuff[0]:          \n",
    "            f[i].extend(kAndas[j][2:])\n",
    "            matched2= 1\n",
    "            ctr2 += 1\n",
    "            break\n",
    "        elif item[2][:-1] == stuff[0] or item[2][:-3] == stuff[0] or item[2][:-1]+'a' == stuff[0] or item[2][:-2]+'a' == stuff[0]:\n",
    "            f[i].extend(kAndas[j][2:])\n",
    "            matched2= 1\n",
    "            ctr2 += 1\n",
    "            break\n",
    "            \n",
    "    if matched2 == 0:\n",
    "        f[i].extend(nanList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "training = pd.read_csv('trainigFiltered.csv')\n",
    "training[(training['tag'] == 'Ds') | (training['tag'] == 'Di')] [['word1','word2']].to_csv('dvandvas.csv',index=False)\n",
    "f = open('dvandvas.csv').read().replace(',','$').splitlines()\n",
    "k = open('dvandvas.dat','w')\n",
    "for item in f:\n",
    "    for chara in item:\n",
    "        print>>k,chara,\n",
    "    print >> k,'\\n'\n",
    "k.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "newTrainingFiltered = open('newHeldoutFiltered.csv','w')\n",
    "for item in f:\n",
    "    print >> newTrainingFiltered,'\\n',\n",
    "    for stuff in item:\n",
    "        print >> newTrainingFiltered,stuff,',',\n",
    "newTrainingFiltered.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filna = 'newHeldoutFiltered.csv'\n",
    "reindex = pd.read_csv(filna).sample(frac=1)\n",
    "del reindex['index']\n",
    "reindex.to_csv('newTrainingFiltered.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}