SVM_Class.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import section\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy\n",
    "import nltk\n",
    "import spacy\n",
    "import gensim\n",
    "import glob\n",
    "import csv\n",
    "from spellchecker import SpellChecker\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import sklearn.model_selection\n",
    "import sklearn.pipeline\n",
    "import re\n",
    "from sklearn import svm\n",
    "from sklearn import *\n",
    "from sklearn.feature_selection import SelectKBest, VarianceThreshold\n",
    "from sklearn.feature_selection import chi2\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "import gensim.models.wrappers.fasttext\n",
    "from scipy import sparse\n",
    "from sklearn.model_selection import LeaveOneOut,KFold,train_test_split\n",
    "\n",
    "\n",
    "# Custom imports\n",
    "from mr_generic_scripts import *\n",
    "from mr_cls_SVM import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path to xlsx files folder\n",
    "path_to_raw_files = 'Data/raw/'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the datasets in dataframes\n",
    "a_dataset = mr_get_data(path_to_raw_files)\n",
    "qa_dataset = mr_get_qa_data(path_to_raw_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Child_ID                                             Answer  Score  Age  \\\n",
      "0         0   Because they did n't want the woman to find them      2    9   \n",
      "1         1           Because they Were n't supose to be there      0    9   \n",
      "2         2                                they have a seccret      0    9   \n",
      "3         3               because they didnt whant to get cort      1    9   \n",
      "4         4  Because they did n't want the woman to see them .      1    9   \n",
      "\n",
      "   Gender           Question  \n",
      "0       0  SFQuestion_1_Text  \n",
      "1       1  SFQuestion_1_Text  \n",
      "2       0  SFQuestion_1_Text  \n",
      "3       0  SFQuestion_1_Text  \n",
      "4       1  SFQuestion_1_Text  \n",
      "   Child_ID                                             Answer  Score  Age  \\\n",
      "0         0  Why did the men hide ? Because they did n't wa...      2    9   \n",
      "1         1  Why did the men hide ? Because they Were n't s...      0    9   \n",
      "2         2         Why did the men hide ? they have a seccret      0    9   \n",
      "3         3  Why did the men hide ? because they didnt whan...      1    9   \n",
      "4         4  Why did the men hide ? Because they did n't wa...      1    9   \n",
      "\n",
      "   Gender           Question  \n",
      "0       0  SFQuestion_1_Text  \n",
      "1       1  SFQuestion_1_Text  \n",
      "2       0  SFQuestion_1_Text  \n",
      "3       0  SFQuestion_1_Text  \n",
      "4       1  SFQuestion_1_Text  \n",
      "1035\n"
     ]
    }
   ],
   "source": [
    "# Sanity check\n",
    "print(a_dataset[0][1][:5])\n",
    "print(qa_dataset[0][1][:5])\n",
    "print(len(a_dataset[0][1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the spacy model\n",
    "spacy_mod = spacy.load(\"en_core_web_sm\") \n",
    "\n",
    "# Def a funciton for basic preprocessing\n",
    "# Input is a string\n",
    "# Output is a sequence of spacy objects\n",
    "def spacy_tok(response,nlp=spacy_mod):\n",
    "\n",
    "    # Linguistic preprocessing\n",
    "    # Ignore missing data, spacy complains\n",
    "    proc_text = nlp(response) if not pd.isna(response) else \"\"\n",
    "    \n",
    "    # Return the spacy objects\n",
    "    return proc_text\n",
    "\n",
    "# Run POS tagging on both input corpora, needed for POS features\n",
    "a_dataset[-1][1][\"Answer\"] = a_dataset[-1][1][\"Answer\"].apply(spacy_tok)\n",
    "qa_dataset[-1][1][\"Answer\"] = qa_dataset[-1][1][\"Answer\"].apply(spacy_tok)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a new instalce of the classifier\n",
    "# We use the predefined column names, age list from 7 to 14\n",
    "# C = 0.1, gamma=\"scale\", kernel = \"rbf\"\n",
    "svm_cls = MR_SVM(text_cols,[7,8,9,10,11,12,13,14],0.1,'scale','rbf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set evaluation parameters - evaluate by question, evaluate by age; do not return incorrect predictions (experimental)\n",
    "svm_cls.mr_set_eval_vars(True,True,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set feature union. No tf-idf. Unigrams and bigrams (tokens), uni-, bi-, tri-grams (chars)\n",
    "svm_cls.ngram_features(False, (1,2), (1,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training\n",
      "Testing the model on the test set:\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Test Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.48 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.48 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.66 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.61 \n",
      "\n",
      "Macro F1: 0.43 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.63 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.57 \n",
      "\n",
      "Macro F1: 0.53 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.4 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.78 \n",
      "\n",
      "Macro F1: 0.65 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.61 \n",
      "\n",
      "Macro F1: 0.53 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.66 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.62 \n",
      "\n",
      "Macro F1: 0.57 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.53 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.65 \n",
      "\n",
      "Macro F1: 0.64 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.66 \n",
      "\n",
      "Macro F1: 0.66 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.65 \n",
      "\n",
      "Macro F1: 0.63 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.74 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 0.5 \n",
      "\n",
      "Macro F1: 0.33 \n",
      "\n",
      "Training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1439: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing the model on the test set:\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Test Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n",
      "Accuracy: 0.65 \n",
      "\n",
      "Macro F1: 0.36 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.62 \n",
      "\n",
      "Macro F1: 0.47 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.49 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.53 \n",
      "\n",
      "Macro F1: 0.5 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.78 \n",
      "\n",
      "Macro F1: 0.49 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.78 \n",
      "\n",
      "Macro F1: 0.61 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.59 \n",
      "\n",
      "Macro F1: 0.56 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.61 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.61 \n",
      "\n",
      "Macro F1: 0.61 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.78 \n",
      "\n",
      "Macro F1: 0.76 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.73 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.65 \n",
      "\n",
      "Macro F1: 0.64 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.66 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.65 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 0.0 \n",
      "\n",
      "Macro F1: 0.0 \n",
      "\n",
      "Training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1439: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing the model on the test set:\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Test Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n",
      "Accuracy: 0.65 \n",
      "\n",
      "Macro F1: 0.36 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.62 \n",
      "\n",
      "Macro F1: 0.46 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.79 \n",
      "\n",
      "Macro F1: 0.66 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.64 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.56 \n",
      "\n",
      "Macro F1: 0.5 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.82 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.66 \n",
      "\n",
      "Macro F1: 0.58 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.8 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.53 \n",
      "\n",
      "Macro F1: 0.51 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.65 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.66 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.76 \n",
      "\n",
      "Macro F1: 0.76 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.65 \n",
      "\n",
      "Macro F1: 0.63 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.56 \n",
      "\n",
      "Training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1439: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing the model on the test set:\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Test Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n",
      "Accuracy: 0.62 \n",
      "\n",
      "Macro F1: 0.41 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.55 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.65 \n",
      "\n",
      "Macro F1: 0.46 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.8 \n",
      "\n",
      "Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.58 \n",
      "\n",
      "Macro F1: 0.54 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.41 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.83 \n",
      "\n",
      "Macro F1: 0.65 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.53 \n",
      "\n",
      "Macro F1: 0.48 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.8 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.49 \n",
      "\n",
      "Macro F1: 0.46 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.45 \n",
      "\n",
      "Macro F1: 0.47 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.6 \n",
      "\n",
      "Macro F1: 0.6 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "Training\n",
      "Testing the model on the test set:\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Test Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n",
      "Accuracy: 0.66 \n",
      "\n",
      "Macro F1: 0.36 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.54 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.55 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.63 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.81 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.64 \n",
      "\n",
      "Macro F1: 0.54 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.83 \n",
      "\n",
      "Macro F1: 0.53 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.59 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.6 \n",
      "\n",
      "Macro F1: 0.49 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.62 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.42 \n",
      "\n",
      "Macro F1: 0.39 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.64 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.66 \n",
      "\n",
      "Macro F1: 0.63 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.66 \n",
      "\n",
      "Macro F1: 0.65 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.43 \n",
      "\n",
      "Training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing the model on the test set:\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Test Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.48 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.65 \n",
      "\n",
      "Macro F1: 0.49 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Macro F1: 0.5 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.61 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.84 \n",
      "\n",
      "Macro F1: 0.77 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.5 \n",
      "\n",
      "Macro F1: 0.45 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.62 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.59 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.57 \n",
      "\n",
      "Macro F1: 0.5 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.8 \n",
      "\n",
      "Macro F1: 0.75 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.55 \n",
      "\n",
      "Macro F1: 0.52 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.77 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.65 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 0.62 \n",
      "\n",
      "Macro F1: 0.61 \n",
      "\n",
      "Training\n",
      "Testing the model on the test set:\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Test Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n",
      "Accuracy: 0.58 \n",
      "\n",
      "Macro F1: 0.37 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.66 \n",
      "\n",
      "Macro F1: 0.39 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.64 \n",
      "\n",
      "Macro F1: 0.53 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.66 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.79 \n",
      "\n",
      "Macro F1: 0.76 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.57 \n",
      "\n",
      "Macro F1: 0.48 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.5 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.79 \n",
      "\n",
      "Macro F1: 0.6 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.65 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.64 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.52 \n",
      "\n",
      "Macro F1: 0.48 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.33 \n",
      "\n",
      "Macro F1: 0.3 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.64 \n",
      "\n",
      "Macro F1: 0.64 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.66 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.66 \n",
      "\n",
      "Macro F1: 0.63 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "Training\n",
      "Testing the model on the test set:\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Test Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n",
      "Accuracy: 0.64 \n",
      "\n",
      "Macro F1: 0.39 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.61 \n",
      "\n",
      "Macro F1: 0.46 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.8 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.66 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.61 \n",
      "\n",
      "Macro F1: 0.57 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.47 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.65 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.61 \n",
      "\n",
      "Macro F1: 0.56 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.79 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.5 \n",
      "\n",
      "Macro F1: 0.48 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.74 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.73 \n",
      "\n",
      "Training\n",
      "Testing the model on the test set:\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Test Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n",
      "Accuracy: 0.61 \n",
      "\n",
      "Macro F1: 0.41 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.55 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.76 \n",
      "\n",
      "Macro F1: 0.54 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.65 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.82 \n",
      "\n",
      "Macro F1: 0.76 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.53 \n",
      "\n",
      "Macro F1: 0.49 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.82 \n",
      "\n",
      "Macro F1: 0.56 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.75 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.61 \n",
      "\n",
      "Macro F1: 0.54 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.79 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.55 \n",
      "\n",
      "Macro F1: 0.54 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.81 \n",
      "\n",
      "Macro F1: 0.8 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.6 \n",
      "\n",
      "Macro F1: 0.6 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.76 \n",
      "\n",
      "Macro F1: 0.75 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "Training\n",
      "Testing the model on the test set:\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Test Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n",
      "Accuracy: 0.66 \n",
      "\n",
      "Macro F1: 0.4 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.53 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Macro F1: 0.64 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.63 \n",
      "\n",
      "Macro F1: 0.58 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.78 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.5 \n",
      "\n",
      "Macro F1: 0.43 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.47 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.81 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.63 \n",
      "\n",
      "Macro F1: 0.54 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.6 \n",
      "\n",
      "Macro F1: 0.55 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.43 \n",
      "\n",
      "Macro F1: 0.47 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 10 fold cross validation, original dataset, without the question\n",
    "results_1 = svm_cls.mr_kfold_train_test(a_dataset[-1][1],10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[   (   [   0.68,\n",
      "            [0.7, 0.68, 0.77, 0.61, 0.71, 0.57, 0.67, 0.78, 0.61, 0.77, 0.62],\n",
      "            [0.67, 0.65, 0.67, 0.66, 0.71, 0.65, 0.75, 0.5]],\n",
      "        [   0.68,\n",
      "            [0.48, 0.48, 0.66, 0.43, 0.63, 0.53, 0.4, 0.65, 0.53, 0.66, 0.57],\n",
      "            [0.53, 0.64, 0.67, 0.66, 0.71, 0.63, 0.74, 0.33]]),\n",
      "    (   [   0.68,\n",
      "            [0.65, 0.62, 0.72, 0.73, 0.77, 0.53, 0.78, 0.78, 0.59, 0.74, 0.61],\n",
      "            [0.78, 0.75, 0.68, 0.68, 0.65, 0.69, 0.69, 0.0]],\n",
      "        [   0.68,\n",
      "            [0.36, 0.47, 0.49, 0.71, 0.7, 0.5, 0.49, 0.61, 0.56, 0.61, 0.61],\n",
      "            [0.76, 0.73, 0.68, 0.68, 0.64, 0.66, 0.65, 0.0]]),\n",
      "    (   [   0.71,\n",
      "            [0.65, 0.62, 0.79, 0.69, 0.75, 0.56, 0.85, 0.82, 0.66, 0.87, 0.53],\n",
      "            [0.67, 0.67, 0.76, 0.69, 0.71, 0.65, 0.69, 0.67]],\n",
      "        [   0.7,\n",
      "            [0.36, 0.46, 0.66, 0.64, 0.7, 0.5, 0.68, 0.69, 0.58, 0.8, 0.51],\n",
      "            [0.65, 0.66, 0.76, 0.68, 0.7, 0.63, 0.68, 0.56]]),\n",
      "    (   [   0.68,\n",
      "            [0.62, 0.68, 0.65, 0.75, 0.8, 0.58, 0.74, 0.83, 0.53, 0.85, 0.49],\n",
      "            [0.45, 0.6, 0.67, 0.68, 0.7, 0.71, 0.72, 1.0]],\n",
      "        [   0.68,\n",
      "            [0.41, 0.55, 0.46, 0.69, 0.72, 0.54, 0.41, 0.65, 0.48, 0.8, 0.46],\n",
      "            [0.47, 0.6, 0.67, 0.67, 0.7, 0.7, 0.7, 1.0]]),\n",
      "    (   [   0.69,\n",
      "            [0.66, 0.71, 0.72, 0.72, 0.81, 0.64, 0.83, 0.77, 0.6, 0.74, 0.42],\n",
      "            [0.69, 0.71, 0.68, 0.7, 0.71, 0.66, 0.66, 0.75]],\n",
      "        [   0.68,\n",
      "            [0.36, 0.54, 0.55, 0.63, 0.7, 0.54, 0.53, 0.59, 0.49, 0.62, 0.39],\n",
      "            [0.64, 0.71, 0.68, 0.69, 0.7, 0.63, 0.65, 0.43]]),\n",
      "    (   [   0.7,\n",
      "            [0.7, 0.65, 0.73, 0.74, 0.84, 0.5, 0.85, 0.72, 0.57, 0.8, 0.55],\n",
      "            [0.77, 0.7, 0.69, 0.68, 0.73, 0.73, 0.68, 0.62]],\n",
      "        [   0.69,\n",
      "            [0.48, 0.49, 0.5, 0.61, 0.77, 0.45, 0.62, 0.59, 0.5, 0.75, 0.52],\n",
      "            [0.77, 0.69, 0.69, 0.67, 0.72, 0.7, 0.65, 0.61]]),\n",
      "    (   [   0.67,\n",
      "            [0.58, 0.66, 0.64, 0.69, 0.79, 0.57, 0.74, 0.79, 0.71, 0.7, 0.52],\n",
      "            [0.33, 0.72, 0.64, 0.67, 0.7, 0.72, 0.66, 1.0]],\n",
      "        [   0.67,\n",
      "            [0.37, 0.39, 0.53, 0.66, 0.76, 0.48, 0.5, 0.6, 0.65, 0.64, 0.48],\n",
      "            [0.3, 0.72, 0.64, 0.66, 0.69, 0.7, 0.63, 1.0]]),\n",
      "    (   [   0.69,\n",
      "            [0.64, 0.61, 0.8, 0.69, 0.77, 0.61, 0.77, 0.85, 0.61, 0.79, 0.5],\n",
      "            [0.67, 0.74, 0.68, 0.67, 0.69, 0.7, 0.71, 0.75]],\n",
      "        [   0.69,\n",
      "            [0.39, 0.46, 0.71, 0.66, 0.72, 0.57, 0.47, 0.65, 0.56, 0.71, 0.48],\n",
      "            [0.72, 0.74, 0.68, 0.67, 0.69, 0.69, 0.68, 0.73]]),\n",
      "    (   [   0.7,\n",
      "            [0.61, 0.72, 0.76, 0.69, 0.82, 0.53, 0.82, 0.85, 0.61, 0.79, 0.55],\n",
      "            [0.81, 0.6, 0.7, 0.68, 0.76, 0.69, 0.73, 1.0]],\n",
      "        [   0.7,\n",
      "            [0.41, 0.55, 0.54, 0.65, 0.76, 0.49, 0.56, 0.75, 0.54, 0.67, 0.54],\n",
      "            [0.8, 0.6, 0.7, 0.67, 0.75, 0.68, 0.72, 1.0]]),\n",
      "    (   [   0.68,\n",
      "            [0.66, 0.7, 0.73, 0.63, 0.78, 0.5, 0.74, 0.81, 0.63, 0.71, 0.6],\n",
      "            [0.43, 0.72, 0.67, 0.67, 0.7, 0.68, 0.71, 1.0]],\n",
      "        [   0.68,\n",
      "            [0.4, 0.53, 0.64, 0.58, 0.68, 0.43, 0.47, 0.69, 0.54, 0.67, 0.55],\n",
      "            [0.47, 0.72, 0.68, 0.67, 0.69, 0.67, 0.68, 1.0]])]\n"
     ]
    }
   ],
   "source": [
    "import pprint\n",
    "pp = pprint.PrettyPrinter(indent=4)\n",
    "pp.pprint(results_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mr_proc_results(raw_results):\n",
    "    # Process the results from the 10 runs\n",
    "    # result format: [acc, acc per q, acc per age], [f1, f1 per q, f1 per age]\n",
    "    # Ignore ages as they seem to be mostly consistent with global average\n",
    "    # Ignore accs per question and age as averaging them seems to be consistent with global average\n",
    "    # Report global acc, global macro f1, average of macro f1 per question\n",
    "    pr_results = [[acc_score, f1_score,round(sum(qa_s)/11,2),round(sum(qf_s)/11,2)] \n",
    "                for ([acc_score, qa_s, aa_s], [f1_score, qf_s, af_s]) in raw_results]\n",
    "    \n",
    "    # Throw the list in an np array\n",
    "    pr_arr = np.array(pr_results)\n",
    "\n",
    "    # Print the results\n",
    "    pp = pprint.PrettyPrinter(indent=4)\n",
    "\n",
    "    print(\"Acc, F1, Avg-F1-Per-Q\")\n",
    "    pp.pprint(pr_results)\n",
    "    pp.pprint(np.mean(pr_arr,axis=0))\n",
    "    \n",
    "    \n",
    "    print(\"\\nAcc per question (avg over 10 runs)\")\n",
    "    # Get the result per question\n",
    "    pr_qa_results = [[qa_s] for ([acc_score, qa_s, aa_s], [f1_score, qf_s, af_s]) in raw_results]\n",
    "    pr_qa_arr = np.array(pr_qa_results)\n",
    "    avg_qa = np.mean(pr_qa_arr,axis=0).tolist()\n",
    "    print(' & '.join([str(round(res,2)) for res in avg_qa[0]]))\n",
    "    #pp.pprint(np.mean(pr_qa_arr,axis=0))\n",
    "    \n",
    "    print(\"\\nF1 per question (avg over 10 runs)\")\n",
    "    pr_qf_results = [[qf_s] for ([acc_score, qa_s, aa_s], [f1_score, qf_s, af_s]) in raw_results]\n",
    "    pr_qf_arr = np.array(pr_qf_results)\n",
    "    avg_qf = np.mean(pr_qf_arr,axis=0).tolist()\n",
    "    print(' & '.join([str(round(res,2)) for res in avg_qf[0]]))\n",
    "    #pp.pprint(np.mean(pr_qf_arr,axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc, F1, Avg-F1-Per-Q\n",
      "[   [0.68, 0.68, 0.68, 0.55],\n",
      "    [0.68, 0.68, 0.68, 0.56],\n",
      "    [0.71, 0.7, 0.71, 0.6],\n",
      "    [0.68, 0.68, 0.68, 0.56],\n",
      "    [0.69, 0.68, 0.69, 0.54],\n",
      "    [0.7, 0.69, 0.7, 0.57],\n",
      "    [0.67, 0.67, 0.67, 0.55],\n",
      "    [0.69, 0.69, 0.69, 0.58],\n",
      "    [0.7, 0.7, 0.7, 0.59],\n",
      "    [0.68, 0.68, 0.68, 0.56]]\n",
      "array([0.688, 0.685, 0.688, 0.566])\n",
      "\n",
      "Acc per question (avg over 10 runs)\n",
      "0.65 & 0.67 & 0.73 & 0.69 & 0.78 & 0.56 & 0.78 & 0.8 & 0.61 & 0.78 & 0.54\n",
      "\n",
      "F1 per question (avg over 10 runs)\n",
      "0.4 & 0.49 & 0.57 & 0.63 & 0.71 & 0.5 & 0.51 & 0.65 & 0.54 & 0.69 & 0.51\n"
     ]
    }
   ],
   "source": [
    "mr_proc_results(results_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training\n",
      "Testing the model on the test set:\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Test Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.29 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.4 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.49 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.64 \n",
      "\n",
      "Macro F1: 0.43 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.51 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.62 \n",
      "\n",
      "Macro F1: 0.42 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.79 \n",
      "\n",
      "Macro F1: 0.38 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.51 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.6 \n",
      "\n",
      "Macro F1: 0.52 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.57 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.78 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.47 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.65 \n",
      "\n",
      "Macro F1: 0.64 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.76 \n",
      "\n",
      "Macro F1: 0.76 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.66 \n",
      "\n",
      "Macro F1: 0.65 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.75 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 0.5 \n",
      "\n",
      "Macro F1: 0.33 \n",
      "\n",
      "Training\n",
      "Testing the model on the test set:\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Test Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.28 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.65 \n",
      "\n",
      "Macro F1: 0.36 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.46 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.63 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.48 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.52 \n",
      "\n",
      "Macro F1: 0.37 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.81 \n",
      "\n",
      "Macro F1: 0.35 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.84 \n",
      "\n",
      "Macro F1: 0.56 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.58 \n",
      "\n",
      "Macro F1: 0.53 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Macro F1: 0.57 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.78 \n",
      "\n",
      "Macro F1: 0.78 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.66 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.64 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 0.0 \n",
      "\n",
      "Macro F1: 0.0 \n",
      "\n",
      "Training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1439: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing the model on the test set:\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Test Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.34 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.62 \n",
      "\n",
      "Macro F1: 0.35 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.51 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.64 \n",
      "\n",
      "Macro F1: 0.44 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.51 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.62 \n",
      "\n",
      "Macro F1: 0.42 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.84 \n",
      "\n",
      "Macro F1: 0.36 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.84 \n",
      "\n",
      "Macro F1: 0.57 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.6 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.53 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.83 \n",
      "\n",
      "Macro F1: 0.8 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.76 \n",
      "\n",
      "Macro F1: 0.76 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Macro F1: 0.73 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.64 \n",
      "\n",
      "Macro F1: 0.63 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.56 \n",
      "\n",
      "Training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1439: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing the model on the test set:\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Test Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.27 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.45 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.54 \n",
      "\n",
      "Macro F1: 0.36 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Macro F1: 0.64 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.52 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.6 \n",
      "\n",
      "Macro F1: 0.44 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.33 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.79 \n",
      "\n",
      "Macro F1: 0.53 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.57 \n",
      "\n",
      "Macro F1: 0.45 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.53 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.62 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.64 \n",
      "\n",
      "Macro F1: 0.62 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.66 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.65 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "Training\n",
      "Testing the model on the test set:\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Test Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.28 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.41 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.46 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.57 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.78 \n",
      "\n",
      "Macro F1: 0.55 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.48 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.82 \n",
      "\n",
      "Macro F1: 0.3 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.83 \n",
      "\n",
      "Macro F1: 0.56 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.63 \n",
      "\n",
      "Macro F1: 0.48 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.51 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.63 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.75 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.77 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Macro F1: 0.73 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.33 \n",
      "\n",
      "Training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1439: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing the model on the test set:\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Test Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.51 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.65 \n",
      "\n",
      "Macro F1: 0.36 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.52 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.53 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.51 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.55 \n",
      "\n",
      "Macro F1: 0.39 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.45 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.8 \n",
      "\n",
      "Macro F1: 0.55 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.59 \n",
      "\n",
      "Macro F1: 0.53 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Macro F1: 0.59 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.74 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.74 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 0.38 \n",
      "\n",
      "Macro F1: 0.28 \n",
      "\n",
      "Training\n",
      "Testing the model on the test set:\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Test Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n",
      "Accuracy: 0.63 \n",
      "\n",
      "Macro F1: 0.31 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.36 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.65 \n",
      "\n",
      "Macro F1: 0.45 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.64 \n",
      "\n",
      "Macro F1: 0.49 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.51 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.66 \n",
      "\n",
      "Macro F1: 0.47 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.78 \n",
      "\n",
      "Macro F1: 0.38 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.79 \n",
      "\n",
      "Macro F1: 0.75 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.65 \n",
      "\n",
      "Macro F1: 0.49 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.76 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.5 \n",
      "\n",
      "Macro F1: 0.52 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.78 \n",
      "\n",
      "Macro F1: 0.77 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.66 \n",
      "\n",
      "Macro F1: 0.66 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.73 \n",
      "\n",
      "Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.66 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "Training\n",
      "Testing the model on the test set:\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Test Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.36 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.59 \n",
      "\n",
      "Macro F1: 0.31 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.78 \n",
      "\n",
      "Macro F1: 0.54 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.65 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.49 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.61 \n",
      "\n",
      "Macro F1: 0.49 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.79 \n",
      "\n",
      "Macro F1: 0.3 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.8 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.61 \n",
      "\n",
      "Macro F1: 0.56 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.53 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.66 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.58 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.74 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.73 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.74 \n",
      "\n",
      "Training\n",
      "Testing the model on the test set:\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Test Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.64 \n",
      "\n",
      "Macro F1: 0.44 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.63 \n",
      "\n",
      "Macro F1: 0.37 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.78 \n",
      "\n",
      "Macro F1: 0.55 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.66 \n",
      "\n",
      "Macro F1: 0.58 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.51 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.57 \n",
      "\n",
      "Macro F1: 0.42 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.49 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.79 \n",
      "\n",
      "Macro F1: 0.62 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.64 \n",
      "\n",
      "Macro F1: 0.56 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.51 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.62 \n",
      "\n",
      "Macro F1: 0.61 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.66 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.71 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.74 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.68 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.73 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.78 \n",
      "\n",
      "Training\n",
      "Testing the model on the test set:\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Test Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.76 \n",
      "\n",
      "Macro F1: 0.36 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n",
      "Accuracy: 0.64 \n",
      "\n",
      "Macro F1: 0.4 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n",
      "Accuracy: 0.62 \n",
      "\n",
      "Macro F1: 0.38 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n",
      "Accuracy: 0.64 \n",
      "\n",
      "Macro F1: 0.46 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.48 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n",
      "Accuracy: 0.61 \n",
      "\n",
      "Macro F1: 0.43 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n",
      "Accuracy: 0.79 \n",
      "\n",
      "Macro F1: 0.39 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n",
      "Accuracy: 0.81 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.57 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.5 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n",
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Age with value 7\n",
      "Accuracy: 0.57 \n",
      "\n",
      "Macro F1: 0.52 \n",
      "\n",
      "Evaluating column Age with value 8\n",
      "Accuracy: 0.78 \n",
      "\n",
      "Macro F1: 0.77 \n",
      "\n",
      "Evaluating column Age with value 9\n",
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.67 \n",
      "\n",
      "Evaluating column Age with value 10\n",
      "Accuracy: 0.7 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Age with value 11\n",
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.69 \n",
      "\n",
      "Evaluating column Age with value 12\n",
      "Accuracy: 0.74 \n",
      "\n",
      "Macro F1: 0.74 \n",
      "\n",
      "Evaluating column Age with value 13\n",
      "Accuracy: 0.72 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 14\n",
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 10 fold cross validation, original dataset, including the question\n",
    "results_2 = svm_cls.mr_kfold_train_test(qa_dataset[-1][1],10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[   (   [   0.71,\n",
      "            [0.75, 0.69, 0.71, 0.64, 0.72, 0.62, 0.79, 0.75, 0.6, 0.75, 0.78],\n",
      "            [0.67, 0.65, 0.71, 0.68, 0.76, 0.66, 0.77, 0.5]],\n",
      "        [   0.71,\n",
      "            [0.29, 0.4, 0.49, 0.43, 0.51, 0.42, 0.38, 0.51, 0.52, 0.57, 0.71],\n",
      "            [0.47, 0.64, 0.71, 0.68, 0.76, 0.65, 0.75, 0.33]]),\n",
      "    (   [   0.69,\n",
      "            [0.74, 0.65, 0.7, 0.74, 0.67, 0.52, 0.81, 0.84, 0.58, 0.73, 0.68],\n",
      "            [1.0, 0.78, 0.68, 0.7, 0.68, 0.69, 0.69, 0.0]],\n",
      "        [   0.69,\n",
      "            [0.28, 0.36, 0.46, 0.63, 0.48, 0.37, 0.35, 0.56, 0.53, 0.57, 0.67],\n",
      "            [1.0, 0.78, 0.69, 0.7, 0.67, 0.66, 0.64, 0.0]]),\n",
      "    (   [   0.72,\n",
      "            [0.72, 0.62, 0.77, 0.64, 0.69, 0.62, 0.84, 0.84, 0.67, 0.75, 0.74],\n",
      "            [0.83, 0.7, 0.76, 0.73, 0.69, 0.64, 0.7, 0.67]],\n",
      "        [   0.71,\n",
      "            [0.34, 0.35, 0.51, 0.44, 0.51, 0.42, 0.36, 0.57, 0.6, 0.53, 0.69],\n",
      "            [0.8, 0.69, 0.76, 0.73, 0.69, 0.63, 0.68, 0.56]]),\n",
      "    (   [   0.69,\n",
      "            [0.69, 0.69, 0.54, 0.73, 0.75, 0.6, 0.77, 0.79, 0.57, 0.75, 0.72],\n",
      "            [0.64, 0.68, 0.68, 0.7, 0.71, 0.69, 0.67, 1.0]],\n",
      "        [   0.69,\n",
      "            [0.27, 0.45, 0.36, 0.64, 0.52, 0.44, 0.33, 0.53, 0.45, 0.53, 0.62],\n",
      "            [0.62, 0.66, 0.68, 0.7, 0.71, 0.67, 0.65, 1.0]]),\n",
      "    (   [   0.73,\n",
      "            [0.7, 0.67, 0.7, 0.71, 0.78, 0.69, 0.82, 0.83, 0.63, 0.74, 0.72],\n",
      "            [0.77, 0.77, 0.72, 0.73, 0.73, 0.73, 0.7, 0.75]],\n",
      "        [   0.72,\n",
      "            [0.28, 0.41, 0.46, 0.57, 0.55, 0.48, 0.3, 0.56, 0.48, 0.51, 0.63],\n",
      "            [0.75, 0.77, 0.71, 0.73, 0.72, 0.72, 0.69, 0.33]]),\n",
      "    (   [   0.72,\n",
      "            [0.75, 0.65, 0.77, 0.68, 0.72, 0.55, 0.85, 0.8, 0.59, 0.73, 0.77],\n",
      "            [0.85, 0.69, 0.71, 0.7, 0.74, 0.7, 0.75, 0.38]],\n",
      "        [   0.71,\n",
      "            [0.51, 0.36, 0.52, 0.53, 0.51, 0.39, 0.45, 0.55, 0.53, 0.59, 0.72],\n",
      "            [0.82, 0.68, 0.71, 0.7, 0.74, 0.68, 0.74, 0.28]]),\n",
      "    (   [   0.7,\n",
      "            [0.63, 0.68, 0.65, 0.64, 0.72, 0.66, 0.78, 0.79, 0.73, 0.65, 0.76],\n",
      "            [0.5, 0.78, 0.66, 0.7, 0.71, 0.73, 0.67, 1.0]],\n",
      "        [   0.7,\n",
      "            [0.31, 0.36, 0.45, 0.49, 0.51, 0.47, 0.38, 0.75, 0.67, 0.49, 0.7],\n",
      "            [0.52, 0.77, 0.66, 0.71, 0.71, 0.72, 0.66, 1.0]]),\n",
      "    (   [   0.71,\n",
      "            [0.7, 0.59, 0.78, 0.71, 0.7, 0.61, 0.79, 0.87, 0.61, 0.75, 0.74],\n",
      "            [0.67, 0.72, 0.68, 0.71, 0.74, 0.72, 0.75, 0.75]],\n",
      "        [   0.71,\n",
      "            [0.36, 0.31, 0.54, 0.65, 0.49, 0.49, 0.3, 0.8, 0.56, 0.53, 0.66],\n",
      "            [0.58, 0.72, 0.68, 0.71, 0.74, 0.71, 0.73, 0.74]]),\n",
      "    (   [   0.71,\n",
      "            [0.64, 0.63, 0.78, 0.66, 0.74, 0.57, 0.89, 0.79, 0.64, 0.77, 0.75],\n",
      "            [0.62, 0.67, 0.72, 0.71, 0.75, 0.68, 0.74, 0.75]],\n",
      "        [   0.71,\n",
      "            [0.44, 0.37, 0.55, 0.58, 0.51, 0.42, 0.49, 0.62, 0.56, 0.51, 0.7],\n",
      "            [0.61, 0.66, 0.71, 0.71, 0.74, 0.67, 0.73, 0.78]]),\n",
      "    (   [   0.7,\n",
      "            [0.76, 0.64, 0.62, 0.64, 0.69, 0.61, 0.79, 0.81, 0.67, 0.7, 0.77],\n",
      "            [0.57, 0.78, 0.67, 0.7, 0.69, 0.74, 0.72, 1.0]],\n",
      "        [   0.7,\n",
      "            [0.36, 0.4, 0.38, 0.46, 0.48, 0.43, 0.39, 0.71, 0.57, 0.5, 0.71],\n",
      "            [0.52, 0.77, 0.67, 0.7, 0.69, 0.74, 0.68, 1.0]])]\n"
     ]
    }
   ],
   "source": [
    "pp = pprint.PrettyPrinter(indent=4)\n",
    "pp.pprint(results_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc, F1, Avg-F1-Per-Q\n",
      "[   [0.71, 0.71, 0.71, 0.48],\n",
      "    [0.69, 0.69, 0.7, 0.48],\n",
      "    [0.72, 0.71, 0.72, 0.48],\n",
      "    [0.69, 0.69, 0.69, 0.47],\n",
      "    [0.73, 0.72, 0.73, 0.48],\n",
      "    [0.72, 0.71, 0.71, 0.51],\n",
      "    [0.7, 0.7, 0.7, 0.51],\n",
      "    [0.71, 0.71, 0.71, 0.52],\n",
      "    [0.71, 0.71, 0.71, 0.52],\n",
      "    [0.7, 0.7, 0.7, 0.49]]\n",
      "array([0.708, 0.705, 0.708, 0.494])\n",
      "\n",
      "Acc per question (avg over 10 runs)\n",
      "0.71 & 0.65 & 0.7 & 0.68 & 0.72 & 0.61 & 0.81 & 0.81 & 0.63 & 0.73 & 0.74\n",
      "\n",
      "F1 per question (avg over 10 runs)\n",
      "0.34 & 0.38 & 0.47 & 0.54 & 0.51 & 0.43 & 0.37 & 0.62 & 0.55 & 0.53 & 0.68\n"
     ]
    }
   ],
   "source": [
    "mr_proc_results(results_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}