classification_using_phoneme_clusters.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification of phonemes using neural signals\n",
    "\n",
    "With a two step process - Bidirectional LSTM and a linear classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################\n",
    "#\n",
    "# 09/03/18 setup\n",
    "#\n",
    "##############################\n",
    "from __future__ import division\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.io\n",
    "from numpy import array\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import torch\n",
    "import random\n",
    "from sklearn.datasets import make_classification\n",
    "#from skorch import NeuralNetClassifier\n",
    "import torch.optim as optim\n",
    "from functools import reduce\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# set random seed\n",
    "# We need to set the cudnn to derministic mode so that \n",
    "#we can get consistent result\n",
    "# Yet, this will slow down the training :( It's a trade-off\n",
    "# see https://discuss.pytorch.org/t/how-to-\n",
    "#confugure-pytorch-to-get-deterministic-results/11797\n",
    "# see https://discuss.pytorch.org/t/network-forward-\n",
    "#backward-calculation-precision-error/17716/2\n",
    "random_state = 100\n",
    "np.random.seed(random_state)\n",
    "torch.manual_seed(random_state)\n",
    "random.seed(random_state)\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print ('using device ' + str(device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parameters\n",
    "data_root_dir = '/Users/janaki/Dropbox/bci/bill_grant_Fall2018'\n",
    "f_name = 'data5_nn_zscored.csv'\n",
    "\n",
    "# load the feature and labels\n",
    "df = pd.read_csv('%s/%s' %(data_root_dir, f_name), header=-1)\n",
    "\n",
    "# define label and feature; convert it to ndarray\n",
    "# adjust the label from 0 to 9\n",
    "# y does not have to be one-hot embedding, i.e., if 10 class, then dim is 1000x10, can be 1000,\n",
    "df_label = df.iloc[:-1,0]\n",
    "df_f = df.iloc[:-1,1:]\n",
    "X1 = df_f.as_matrix()\n",
    "y1 = df_label.as_matrix() \n",
    "\n",
    "print(y1.shape)\n",
    "print(X1.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#converting data into the form required by an LSTM (time, batch size, number of features)\n",
    "X_raw = np.zeros((225,199,267))\n",
    "for i in np.arange(267):\n",
    "    X_raw[:,:,i] = np.reshape(X1[:,i],(199,225)).transpose()\n",
    "    assert np.all(X_raw[:,0,i] == X1[0:225,i])\n",
    "y = np.reshape(y1,(199,225)).transpose()\n",
    "s = np.arange(X_raw.shape[1])\n",
    "np.random.shuffle(s)\n",
    "X_raw = X_raw[:,s,:]\n",
    "X_raw = X_raw.astype(np.float32)\n",
    "y = y[:,s]\n",
    "print(X_raw.shape)\n",
    "print(y.shape)\n",
    "print(X_raw.size)\n",
    "print(y.size)\n",
    "#assert np.all(torch.tensor(y,dtype = torch.long).contiguous().view(225*123) == torch.tensor(y1,dtype = torch.long).contiguous().view(225*123))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate class ratio for loss calculation because it is imbalanced\n",
    "\n",
    "def get_weight_balance(y, amp_weight=1):\n",
    "    \"\"\"\n",
    "    Calculate the class ratio.\n",
    "    \"\"\"\n",
    "\n",
    "    y_labels = np.reshape(y,(y.size,1))\n",
    "    counts_np = np.bincount(y_labels[:,0])\n",
    "    \n",
    "    #counts_np = np.vstack((temp2,temp1[temp2])).T\n",
    "    max_val = np.max(np.abs(counts_np),axis=0)\n",
    "    \n",
    "    class_weight = np.max(np.abs(counts_np),axis=0)/counts_np\n",
    "    class_weight = class_weight*amp_weight\n",
    "    class_weight = class_weight.tolist()\n",
    "    \n",
    "    return class_weight\n",
    "weight = get_weight_balance(y)\n",
    "print(weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Prepare the datasets for classification of clustered classes.\n",
    "\"\"\"\n",
    "\n",
    "def get_dataset(X_raw, y):\n",
    "    \n",
    "    # pull dataset for each class from the X_raw and y sets\n",
    "    e_index = np.where(y == 2)\n",
    "    X_e_vowel = np.squeeze(X_raw[e_index,:])\n",
    "    y_e_vowel = y[e_index]\n",
    "    \n",
    "    i_index = np.where(y == 3)\n",
    "    X_i_vowel = np.squeeze(X_raw[i_index,:])\n",
    "    y_i_vowel = y[i_index]\n",
    "    \n",
    "    o_index = np.where(y == 4)\n",
    "    X_o_vowel = np.squeeze(X_raw[o_index,:])\n",
    "    y_o_vowel = y[o_index]\n",
    "    \n",
    "    u_index = np.where(y == 5)\n",
    "    X_u_vowel = np.squeeze(X_raw[u_index,:])\n",
    "    y_u_vowel = y[u_index]\n",
    "    \n",
    "    m_index = np.where(y == 8)\n",
    "    X_m_conso = np.squeeze(X_raw[m_index,:])\n",
    "    y_m_conso = y[m_index]\n",
    "    \n",
    "    n_index = np.where(y == 9)\n",
    "    X_n_conso = np.squeeze(X_raw[n_index,:])\n",
    "    y_n_conso = y[n_index]\n",
    "    \n",
    "    l_index = np.where(y == 6)\n",
    "    X_l_conso = np.squeeze(X_raw[l_index,:])\n",
    "    y_l_conso = y[l_index]   \n",
    "    \n",
    "    r_index = np.where(y == 7)\n",
    "    X_r_conso = np.squeeze(X_raw[r_index,:])\n",
    "    y_r_conso = y[r_index]    \n",
    "\n",
    "    # create the class sets\n",
    "    ei_length = np.minimum(y_e_vowel.shape[0], y_i_vowel.shape[0])\n",
    "    e_index = random.sample(range(y_e_vowel.shape[0]),ei_length)\n",
    "    i_index = random.sample(range(y_i_vowel.shape[0]),ei_length)\n",
    "    X_e = X_e_vowel[e_index,:]\n",
    "    X_i = X_i_vowel[i_index,:]\n",
    "    y_e = np.zeros(X_e.shape[0])\n",
    "    y_i = np.ones(X_i.shape[0])\n",
    "    \n",
    "    ou_length = np.minimum(y_o_vowel.shape[0], y_u_vowel.shape[0])\n",
    "    o_index = random.sample(range(y_o_vowel.shape[0]),ou_length)\n",
    "    u_index = random.sample(range(y_u_vowel.shape[0]),ou_length)\n",
    "    X_o = X_o_vowel[o_index,:]\n",
    "    X_u = X_u_vowel[u_index,:]\n",
    "    y_o = np.zeros(X_o.shape[0])\n",
    "    y_u = np.ones(X_u.shape[0])\n",
    "    \n",
    "    mn_length = np.minimum(y_m_conso.shape[0], y_n_conso.shape[0])\n",
    "    m_index = random.sample(range(y_m_conso.shape[0]),mn_length)\n",
    "    n_index = random.sample(range(y_n_conso.shape[0]),mn_length)\n",
    "    X_m = X_m_conso[m_index,:]\n",
    "    X_n = X_n_conso[n_index,:]\n",
    "    y_m = np.zeros(X_m.shape[0])\n",
    "    y_n = np.ones(X_n.shape[0])    \n",
    "    \n",
    "    lr_length = np.minimum(y_l_conso.shape[0], y_r_conso.shape[0])\n",
    "    l_index = random.sample(range(y_l_conso.shape[0]),lr_length)\n",
    "    r_index = random.sample(range(y_r_conso.shape[0]),lr_length)\n",
    "    X_l = X_l_conso[l_index,:]\n",
    "    X_r = X_r_conso[r_index,:]\n",
    "    y_l = np.zeros(X_l.shape[0])\n",
    "    y_r = np.ones(X_r.shape[0])    \n",
    "    \n",
    "    \n",
    "    X_ei = np.concatenate((X_e, X_i))\n",
    "    y_ei = np.concatenate((y_e, y_i))\n",
    "    X_ou = np.concatenate((X_o, X_u))\n",
    "    y_ou = np.concatenate((y_o, y_u))\n",
    "    X_mn = np.concatenate((X_m, X_n))\n",
    "    y_mn = np.concatenate((y_m, y_n))\n",
    "    X_lr = np.concatenate((X_l, X_r))\n",
    "    y_lr = np.concatenate((y_l, y_r))\n",
    "    #import pdb; pdb.set_trace()\n",
    "    \n",
    "    return X_ei,y_ei,X_ou,y_ou,X_mn,y_mn,X_lr,y_lr\n",
    "\n",
    "# check\n",
    "X_ei,y_ei,X_ou,y_ou,X_mn,y_mn,X_lr,y_lr = get_dataset(X1,y1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Define the RNN model.\n",
    "\"\"\"\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class RNN(nn.Module):\n",
    "    def __init__(self, num_directions, num_layers, hidden_dim, vocab_size, num_chan, batch_size, prob):\n",
    "        super(RNN, self).__init__()\n",
    "        \n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.vocab_size = vocab_size\n",
    "        self.direction = num_directions\n",
    "        self.batch_size = batch_size\n",
    "        self.num_layers = num_layers\n",
    "                \n",
    "        #first layer lstm cell\n",
    "        self.lstm = nn.LSTM(input_size = num_chan, hidden_size=hidden_dim, \n",
    "                            bidirectional = self.direction>1, num_layers = self.num_layers)  \n",
    "        self.affine0 = nn.Linear(in_features = num_directions*hidden_dim, \n",
    "                                 out_features = vocab_size)\n",
    "        self.softmax = nn.LogSoftmax(dim=2)\n",
    "        self.dropout = nn.Dropout(p=prob)\n",
    "\n",
    "    def forward(self, X, hc, **kwargs):\n",
    "        \"\"\"\n",
    "            x: input to the model\n",
    "                *  x[t] - input of shape (batch, input_size) at time t\n",
    "                \n",
    "            hc: hidden and cell states\n",
    "                *  tuple of hidden and cell state\n",
    "        \"\"\" \n",
    "        hc_1 = hc\n",
    "        output_lstm, (hn, cn) = self.lstm(X, hc_1)\n",
    "        output_seq = self.affine0(output_lstm)\n",
    "        \n",
    "        # return the output sequence\n",
    "        #pdb.set_trace()  \n",
    "        #view goes across each row and then traverses down the matrix\n",
    "        return output_seq.view((X.shape[0]*self.batch_size, self.vocab_size))\n",
    "    \n",
    "    def initHidden(self,device = None):\n",
    "        \n",
    "        # initialize the hidden state and the cell state to zeros\n",
    "        #pdb.set_trace()\n",
    "\n",
    "        return (torch.zeros((self.direction*self.num_layers, self.batch_size, self.hidden_dim), \n",
    "                            dtype=torch.float32, device=device),\n",
    "                torch.zeros((self.direction*self.num_layers, self.batch_size, self.hidden_dim), \n",
    "                            dtype=torch.float32, device=device)) \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Train the RNN model\n",
    "\"\"\"\n",
    "def train_rnn(rnn, input_tensor, target_tensor):\n",
    "    hidden = rnn.initHidden(device = device)\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "    output = rnn(input_tensor, hidden)\n",
    "\n",
    "    loss = criterion(output, target_tensor)\n",
    "    \n",
    "    # calculate the gradients\n",
    "    loss.backward()\n",
    "\n",
    "    # update the parameters of the model\n",
    "    optimizer.step()\n",
    "\n",
    "    return output, loss.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def random_samples_without_zero(X, y, n):\n",
    "    samples_without_zero = []\n",
    "        samples_without_zero.append ([X[:,n,:],y[:,n]])\n",
    "        #import pdb; pdb.set_trace()\n",
    "    return  samples_without_zero\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Test the model\n",
    "\"\"\"\n",
    "def evaluate_rnn(input_tensor):\n",
    "    rnn.batch_size = 1\n",
    "    hidden = rnn.initHidden(device = device)\n",
    "    output = rnn(input_tensor, hidden)\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "t0 = time.time()\n",
    "\n",
    "# define the input data\n",
    "X_raw = X_raw.astype(np.float32) #225x199x178\n",
    "y = y.astype(np.int64) #225x199\n",
    "y_copy = np.copy(y)\n",
    "\n",
    "#define output variables\n",
    "out_proba_mat = np.zeros((225,199,7))\n",
    "val_mat = np.zeros_like(y)\n",
    "y_new = np.zeros_like(y)\n",
    "val_accuracy = np.zeros(y.shape[1])\n",
    "\n",
    "#converting to torch variables\n",
    "y_gpu = torch.tensor(y_copy, dtype=torch.long, device=device)\n",
    "X_raw_gpu = torch.tensor(X_raw, device=device)\n",
    "\n",
    "#defining parameters\n",
    "n_hidden = 128\n",
    "\n",
    "#variables for visualization\n",
    "print_every = 30\n",
    "plot_every = 1\n",
    "val_every = 1\n",
    "current_tr_loss = 0\n",
    "current_val_loss = 0\n",
    "val_loss = 0\n",
    "cnt = 0 \n",
    "all_tr_losses = []\n",
    "all_val_losses = []\n",
    "itr = 30\n",
    "\n",
    "#To simplify the problem, of not classifying all phoneme classes, but clustering them by voicing and tongue position.\n",
    "y_copy[np.where(y == 2)] = 2\n",
    "y_copy[np.where(y == 3)] = 2\n",
    "y_copy[np.where(y == 4)] = 3\n",
    "y_copy[np.where(y == 5)] = 3\n",
    "y_copy[np.where(y == 9)] = 4\n",
    "y_copy[np.where(y == 6)] = 4 \n",
    "y_copy[np.where(y == 7)] = 5\n",
    "y_copy[np.where(y == 10)] = 5\n",
    "y_copy[np.where(y == 8)] = 6\n",
    "\n",
    "#linear classification within a class\n",
    "X_ei,y_ei,X_ou,y_ou,X_mn,y_mn,X_lr,y_lr = get_dataset(X1,y1)\n",
    "clf_ei = LinearDiscriminantAnalysis()#LinearSVC(random_state = 0, tol = 1e-5, max_iter = 5000)\n",
    "clf_ou = LinearDiscriminantAnalysis()#LinearSVC(random_state = 0, tol = 1e-5, max_iter = 5000)\n",
    "clf_mn = LinearDiscriminantAnalysis()#LinearSVC(random_state = 0, tol = 1e-5, max_iter = 5000)\n",
    "clf_lr = LinearDiscriminantAnalysis()#LinearSVC(random_state = 0, tol = 1e-5, max_iter = 5000)\n",
    "clf_ei.fit(X_ei, y_ei)\n",
    "clf_ou.fit(X_ou, y_ou)\n",
    "clf_mn.fit(X_mn, y_mn)\n",
    "clf_lr.fit(X_lr, y_lr)\n",
    "\n",
    "for test_index in np.arange(y.shape[1]):\n",
    "    train_index = np.delete(np.arange(y.shape[1]),test_index)\n",
    "\n",
    "    val_samples =  random_samples_without_zero(X_raw_gpu,y_gpu,test_index)\n",
    "    \n",
    "    #assigning weights to each class\n",
    "    class_weight_samples = y_copy[:,train_index]\n",
    "    class_weight = get_weight_balance(class_weight_samples, amp_weight=1)\n",
    "    class_weight = torch.tensor(class_weight, device = device, dtype=torch.float32)\n",
    "   \n",
    "    rnn = RNN(num_layers = 1, num_directions = 2, hidden_dim = n_hidden, vocab_size = 11, num_chan = 267, batch_size = len(train_index), prob = 0)  \n",
    "    rnn = rnn.to(device)\n",
    "    criterion = nn.CrossEntropyLoss(weight = class_weight)\n",
    "    optimizer = optim.Adam(rnn.parameters(), lr=0.005)\n",
    "\n",
    "    #training step\n",
    "    for iter in np.arange(itr):\n",
    "        input_sample =  random_samples_without_zero(X_raw_gpu,y_gpu,train_index)\n",
    "        # massaging the input data\n",
    "        input_y = input_sample[0][1].view(225*len(train_index))\n",
    "        input_X = input_sample[0][0]\n",
    "        \n",
    "        output, loss = train_rnn(rnn, input_X, input_y)\n",
    "        current_tr_loss += loss\n",
    "        cnt += 1\n",
    "\n",
    "        # Print iter number, loss, name and guess\n",
    "        if cnt % print_every == 0:\n",
    "            out_proba_val = F.softmax(output, dim=-1)  \n",
    "            out_proba_val = out_proba_val.cpu().detach().numpy()\n",
    "            #print(np.argmax(out_proba_val,axis=1))\n",
    "            #print(input_y)\n",
    "\n",
    "        if iter % plot_every == 0:\n",
    "            all_tr_losses.append(current_tr_loss)\n",
    "            current_tr_loss = 0\n",
    "            #print(all_tr_losses)\n",
    "\n",
    "    # validation step\n",
    "    val_X = val_samples[0][0].unsqueeze(1)\n",
    "    val_y = val_samples[0][1] \n",
    "\n",
    "    output = evaluate_rnn(val_X)\n",
    "    out_proba_val = F.softmax(output, dim=-1)  \n",
    "    out_proba_val = out_proba_val.cpu().detach().numpy()\n",
    "    \n",
    "    val_X_cpu = np.squeeze(X_raw[:,test_index,:])\n",
    "    \n",
    "    #Multiplication of probabilities\n",
    "    for vl in np.arange(out_proba_val.shape[0]):\n",
    "        arg_val = np.argmax(out_proba_val[vl,:])\n",
    "        if arg_val == 2:\n",
    "            prob_ei = clf_ei.predict_proba(np.expand_dims(val_X_cpu[vl,:], axis = 0))\n",
    "            out_proba_mat[vl,test_index,2] = out_proba_val[vl,2]*prob_ei[0][0]\n",
    "            out_proba_mat[vl,test_index,3] = out_proba_val[vl,2]*prob_ei[0][1]\n",
    "        elif arg_val == 3:\n",
    "            prob_ou = clf_ou.predict_proba(np.expand_dims(val_X_cpu[vl,:], axis = 0))\n",
    "            out_proba_mat[vl,test_index,4] = out_proba_val[vl,3]*prob_ou[0][0]\n",
    "            out_proba_mat[vl,test_index,5] = out_proba_val[vl,3]*prob_ou[0][1]\n",
    "        elif arg_val == 4:  \n",
    "            prob_mn = clf_mn.predict_proba(np.expand_dims(val_X_cpu[vl,:], axis = 0))\n",
    "            out_proba_mat[vl,test_index,8] = out_proba_val[vl,4]*prob_mn[0][0]\n",
    "            out_proba_mat[vl,test_index,9] = out_proba_val[vl,4]*prob_mn[0][1]\n",
    "        elif arg_val == 5:\n",
    "            prob_lr = clf_lr.predict_proba(np.expand_dims(val_X_cpu[vl,:], axis = 0))\n",
    "            out_proba_mat[vl,test_index,6] = out_proba_val[vl,5]*prob_lr[0][0]\n",
    "            out_proba_mat[vl,test_index,7] = out_proba_val[vl,5]*prob_lr[0][1]\n",
    "        elif arg_val == 6:\n",
    "            out_proba_mat[vl,test_index,10] = out_proba_val[vl,6]\n",
    "        else:\n",
    "            out_proba_mat[vl,test_index,arg_val] = out_proba_val[vl,arg_val]\n",
    "            \n",
    "    out_proba_mat[:,test_index,:] = out_proba_val\n",
    "    val_preds = np.argmax(out_proba_val,axis=1)\n",
    "    val_mat[:,test_index] = val_preds\n",
    "    y_new[:,test_index] = val_y.cpu().detach().numpy()\n",
    "    plt.plot(all_tr_losses)\n",
    "    plt.show()\n",
    "    print(val_y)\n",
    "    print(val_preds)\n",
    "t1 = time.time()\n",
    "total = t1-t0    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_col = y_new.reshape(y_new.size,1)\n",
    "val_col = val_mat.reshape(val_mat.size,1)\n",
    "\n",
    "\"\"\"\n",
    "Evaluate the performance\n",
    "\"\"\"\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "# calculation the confusion matrix\n",
    "# row: true label\n",
    "# col: predicted value\n",
    "cm = np.array(confusion_matrix(y_col,val_col))\n",
    "#cm = np.transpose(cm)\n",
    "print(df.iloc[:,0].value_counts())\n",
    "freq = np.array ([37577, 2996, 1444, 2060, 2074, 1780, 399, 401, 124, 97, 101])\n",
    "print('confusion matrix')\n",
    "print(cm)\n",
    "print(np.sum(cm.diagonal())/np.sum(freq))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_accuracy = np.mean(val_mat == y_new, axis = 0)\n",
    "print(val_accuracy)\n",
    "print(np.sum(val_accuracy>0.90))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import seaborn as sn\n",
    "\n",
    "# export to csv`\n",
    "df_cm = pd.DataFrame(cm/freq[:,None])\n",
    "sn.set(font_scale=1)\n",
    "accuracies = sn.heatmap(df_cm, annot=True, annot_kws={\"size\":9})\n",
    "#figure = accuracies.get_figure()\n",
    "#figure.savefig('Subject3_rnn.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import scipy.io\n",
    "scipy.io.savemat('/Users/janaki/Dropbox/bci/bill_grant_Fall2018/probabilities_data6_nn_zscored_256_35.mat', mdict={'out_proba_mat': out_proba_mat})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}