Transformer_Class.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy\n",
    "import nltk\n",
    "import spacy\n",
    "import gensim\n",
    "import glob\n",
    "import csv\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import sklearn.model_selection\n",
    "import sklearn.pipeline\n",
    "import re\n",
    "from sklearn import svm\n",
    "from sklearn import *\n",
    "from sklearn.feature_selection import SelectKBest, VarianceThreshold\n",
    "from sklearn.feature_selection import chi2\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "import gensim.models.wrappers.fasttext\n",
    "from scipy import sparse\n",
    "import tensorflow_datasets as tfds\n",
    "import tensorflow as tf\n",
    "import collections\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.model_selection import LeaveOneOut,KFold,train_test_split\n",
    "import ktrain\n",
    "from ktrain import text\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "# Custom imports\n",
    "from mr_generic_scripts import *\n",
    "from mr_cls_Transformer import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path to xlsx files folder\n",
    "path_to_raw_files = 'Data/raw/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the datasets in dataframes\n",
    "a_dataset = mr_get_data(path_to_raw_files)\n",
    "qa_dataset = mr_get_qa_data(path_to_raw_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Child_ID                                             Answer  Score  Age  \\\n",
      "0         0   Because they did n't want the woman to find them      2    9   \n",
      "1         1           Because they Were n't supose to be there      0    9   \n",
      "2         2                                they have a seccret      0    9   \n",
      "3         3               because they didnt whant to get cort      1    9   \n",
      "4         4  Because they did n't want the woman to see them .      1    9   \n",
      "\n",
      "   Gender           Question  \n",
      "0       0  SFQuestion_1_Text  \n",
      "1       1  SFQuestion_1_Text  \n",
      "2       0  SFQuestion_1_Text  \n",
      "3       0  SFQuestion_1_Text  \n",
      "4       1  SFQuestion_1_Text  \n",
      "   Child_ID                                             Answer  Score  Age  \\\n",
      "0         0  Why did the men hide ? Because they did n't wa...      2    9   \n",
      "1         1  Why did the men hide ? Because they Were n't s...      0    9   \n",
      "2         2         Why did the men hide ? they have a seccret      0    9   \n",
      "3         3  Why did the men hide ? because they didnt whan...      1    9   \n",
      "4         4  Why did the men hide ? Because they did n't wa...      1    9   \n",
      "\n",
      "   Gender           Question  \n",
      "0       0  SFQuestion_1_Text  \n",
      "1       1  SFQuestion_1_Text  \n",
      "2       0  SFQuestion_1_Text  \n",
      "3       0  SFQuestion_1_Text  \n",
      "4       1  SFQuestion_1_Text  \n",
      "1035\n"
     ]
    }
   ],
   "source": [
    "# Sanity check\n",
    "print(a_dataset[0][1][:5])\n",
    "print(qa_dataset[0][1][:5])\n",
    "print(len(a_dataset[0][1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_cls = MR_transformer(text_cols,[7,8,9,10,11,12,13,14],[0,1,2],20)\n",
    "tr_cls.mr_set_eval_vars(True,True,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mr_proc_results(raw_results):\n",
    "    # Process the results from the 10 runs\n",
    "    # result format: [acc, acc per q, acc per age], [f1, f1 per q, f1 per age]\n",
    "    # Ignore ages as they seem to be mostly consistent with global average\n",
    "    # Ignore accs per question and age as averaging them seems to be consistent with global average\n",
    "    # Report global acc, global macro f1, average of macro f1 per question\n",
    "    pr_results = [[acc_score, f1_score,round(sum(qa_s)/11,2),round(sum(qf_s)/11,2)] \n",
    "                for ([acc_score, qa_s, aa_s], [f1_score, qf_s, af_s]) in raw_results]\n",
    "    \n",
    "    # Throw the list in an np array\n",
    "    pr_arr = np.array(pr_results)\n",
    "\n",
    "    # Print the results\n",
    "    pp = pprint.PrettyPrinter(indent=4)\n",
    "\n",
    "    print(\"Acc, F1, Avg-F1-Per-Q\")\n",
    "    pp.pprint(pr_results)\n",
    "    pp.pprint(np.mean(pr_arr,axis=0))\n",
    "    \n",
    "    \n",
    "    print(\"\\nAcc per question (avg over 10 runs)\")\n",
    "    # Get the result per question\n",
    "    pr_qa_results = [[qa_s] for ([acc_score, qa_s, aa_s], [f1_score, qf_s, af_s]) in raw_results]\n",
    "    pr_qa_arr = np.array(pr_qa_results)\n",
    "    avg_qa = np.mean(pr_qa_arr,axis=0).tolist()\n",
    "    print(' & '.join([str(round(res,2)) for res in avg_qa[0]]))\n",
    "    #pp.pprint(np.mean(pr_qa_arr,axis=0))\n",
    "    \n",
    "    print(\"\\nF1 per question (avg over 10 runs)\")\n",
    "    pr_qf_results = [[qf_s] for ([acc_score, qa_s, aa_s], [f1_score, qf_s, af_s]) in raw_results]\n",
    "    pr_qf_arr = np.array(pr_qf_results)\n",
    "    avg_qf = np.mean(pr_qf_arr,axis=0).tolist()\n",
    "    print(' & '.join([str(round(res,2)) for res in avg_qf[0]]))\n",
    "    #pp.pprint(np.mean(pr_qf_arr,axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 27\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 26\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 83s 65ms/step - loss: 0.6365 - accuracy: 0.7384 - val_loss: 0.4630 - val_accuracy: 0.8248\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 73s 57ms/step - loss: 0.3629 - accuracy: 0.8714 - val_loss: 0.5685 - val_accuracy: 0.7980\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 73s 57ms/step - loss: 0.2510 - accuracy: 0.9121 - val_loss: 0.3627 - val_accuracy: 0.8853\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 73s 57ms/step - loss: 0.0971 - accuracy: 0.9671 - val_loss: 0.3882 - val_accuracy: 0.8927\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      0.86      0.87       768\n",
      "           1       0.89      0.87      0.88       863\n",
      "           2       0.92      0.94      0.93       914\n",
      "\n",
      "    accuracy                           0.89      2545\n",
      "   macro avg       0.89      0.89      0.89      2545\n",
      "weighted avg       0.89      0.89      0.89      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.89 \n",
      "\n",
      "Test Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.8 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.83 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.84 \n",
      "\n",
      "Macro F1: 0.83 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.82 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.96 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.62 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.81 \n",
      "\n",
      "Macro F1: 0.81 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\ktrain\\text\\preprocessor.py:228: UserWarning: List or array of two texts supplied, so task being treated as text classification. If this is a sentence pair classification task, please cast to tuple.\n",
      "  warnings.warn('List or array of two texts supplied, so task being treated as text classification. ' +\\\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 26\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 26\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 82s 65ms/step - loss: 0.6219 - accuracy: 0.7345 - val_loss: 0.4489 - val_accuracy: 0.8428\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 73s 57ms/step - loss: 0.3552 - accuracy: 0.8756 - val_loss: 0.5123 - val_accuracy: 0.8346\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 73s 57ms/step - loss: 0.2356 - accuracy: 0.9217 - val_loss: 0.3863 - val_accuracy: 0.8664\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 73s 58ms/step - loss: 0.0961 - accuracy: 0.9724 - val_loss: 0.4427 - val_accuracy: 0.8739\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.84      0.84       742\n",
      "           1       0.87      0.87      0.87       921\n",
      "           2       0.90      0.91      0.91       882\n",
      "\n",
      "    accuracy                           0.87      2545\n",
      "   macro avg       0.87      0.87      0.87      2545\n",
      "weighted avg       0.87      0.87      0.87      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.89 \n",
      "\n",
      "Test Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.78 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.64 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.84 \n",
      "\n",
      "Macro F1: 0.83 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.78 \n",
      "\n",
      "Macro F1: 0.74 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.84 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.84 \n",
      "\n",
      "Macro F1: 0.83 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 26\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 21\n",
      "\t99percentile : 27\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 83s 66ms/step - loss: 0.6447 - accuracy: 0.7281 - val_loss: 0.3864 - val_accuracy: 0.8585\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 74s 58ms/step - loss: 0.3709 - accuracy: 0.8665 - val_loss: 0.3655 - val_accuracy: 0.8668\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 74s 58ms/step - loss: 0.2511 - accuracy: 0.9172 - val_loss: 0.3195 - val_accuracy: 0.8927\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 74s 58ms/step - loss: 0.0991 - accuracy: 0.9686 - val_loss: 0.3585 - val_accuracy: 0.8955\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.87      0.86       771\n",
      "           1       0.89      0.89      0.89       868\n",
      "           2       0.93      0.93      0.93       906\n",
      "\n",
      "    accuracy                           0.90      2545\n",
      "   macro avg       0.89      0.89      0.89      2545\n",
      "weighted avg       0.90      0.90      0.90      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.88 \n",
      "\n",
      "Test Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.84 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.78 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.81 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.84 \n",
      "\n",
      "Macro F1: 0.81 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.83 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 26\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 21\n",
      "\t99percentile : 26\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 85s 66ms/step - loss: 0.6408 - accuracy: 0.7362 - val_loss: 0.4206 - val_accuracy: 0.8511\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 74s 58ms/step - loss: 0.3692 - accuracy: 0.8673 - val_loss: 0.3792 - val_accuracy: 0.8625\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 73s 57ms/step - loss: 0.2536 - accuracy: 0.9082 - val_loss: 0.3525 - val_accuracy: 0.8825\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 73s 58ms/step - loss: 0.0889 - accuracy: 0.9704 - val_loss: 0.4131 - val_accuracy: 0.8790\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      0.85      0.86       816\n",
      "           1       0.87      0.85      0.86       858\n",
      "           2       0.89      0.93      0.91       871\n",
      "\n",
      "    accuracy                           0.88      2545\n",
      "   macro avg       0.88      0.88      0.88      2545\n",
      "weighted avg       0.88      0.88      0.88      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.88 \n",
      "\n",
      "Test Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.81 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.81 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.78 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.84 \n",
      "\n",
      "Macro F1: 0.77 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.75 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.83 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.82 \n",
      "\n",
      "Macro F1: 0.81 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.78 \n",
      "\n",
      "Macro F1: 0.78 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 26\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 27\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 83s 65ms/step - loss: 0.6191 - accuracy: 0.7390 - val_loss: 0.4207 - val_accuracy: 0.8460\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 72s 57ms/step - loss: 0.3635 - accuracy: 0.8718 - val_loss: 0.4407 - val_accuracy: 0.8511\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 72s 57ms/step - loss: 0.2439 - accuracy: 0.9208 - val_loss: 0.3852 - val_accuracy: 0.8825\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 73s 57ms/step - loss: 0.1015 - accuracy: 0.9674 - val_loss: 0.3929 - val_accuracy: 0.8923\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.85      0.87       781\n",
      "           1       0.85      0.90      0.87       831\n",
      "           2       0.94      0.93      0.93       933\n",
      "\n",
      "    accuracy                           0.89      2545\n",
      "   macro avg       0.89      0.89      0.89      2545\n",
      "weighted avg       0.89      0.89      0.89      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.89 \n",
      "\n",
      "Test Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.83 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.81 \n",
      "\n",
      "Macro F1: 0.73 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.96 \n",
      "\n",
      "Macro F1: 0.95 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.63 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.83 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.77 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.8 \n",
      "\n",
      "Macro F1: 0.81 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 27\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 26\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 83s 65ms/step - loss: 0.6257 - accuracy: 0.7357 - val_loss: 0.3883 - val_accuracy: 0.8566\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 73s 58ms/step - loss: 0.3623 - accuracy: 0.8727 - val_loss: 0.3841 - val_accuracy: 0.8515\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 73s 58ms/step - loss: 0.2396 - accuracy: 0.9194 - val_loss: 0.3406 - val_accuracy: 0.8750\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 73s 57ms/step - loss: 0.0878 - accuracy: 0.9743 - val_loss: 0.4141 - val_accuracy: 0.8876\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.84      0.86       823\n",
      "           1       0.85      0.89      0.87       814\n",
      "           2       0.92      0.93      0.93       908\n",
      "\n",
      "    accuracy                           0.89      2545\n",
      "   macro avg       0.89      0.89      0.89      2545\n",
      "weighted avg       0.89      0.89      0.89      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.89 \n",
      "\n",
      "Test Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.83 \n",
      "\n",
      "Macro F1: 0.7 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.78 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.81 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.96 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 26\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 26\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 84s 66ms/step - loss: 0.6241 - accuracy: 0.7422 - val_loss: 0.4159 - val_accuracy: 0.8554\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 74s 58ms/step - loss: 0.3679 - accuracy: 0.8710 - val_loss: 0.3787 - val_accuracy: 0.8743\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 74s 59ms/step - loss: 0.2455 - accuracy: 0.9164 - val_loss: 0.3617 - val_accuracy: 0.8747\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 74s 58ms/step - loss: 0.0980 - accuracy: 0.9670 - val_loss: 0.3597 - val_accuracy: 0.8963\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.86      0.88       798\n",
      "           1       0.86      0.89      0.88       875\n",
      "           2       0.93      0.93      0.93       872\n",
      "\n",
      "    accuracy                           0.90      2545\n",
      "   macro avg       0.90      0.90      0.90      2545\n",
      "weighted avg       0.90      0.90      0.90      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.88 \n",
      "\n",
      "Test Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.8 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.83 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.84 \n",
      "\n",
      "Macro F1: 0.81 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.82 \n",
      "\n",
      "Macro F1: 0.78 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.53 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 27\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 27\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 83s 65ms/step - loss: 0.6204 - accuracy: 0.7395 - val_loss: 0.4112 - val_accuracy: 0.8444\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 73s 57ms/step - loss: 0.3621 - accuracy: 0.8733 - val_loss: 0.4197 - val_accuracy: 0.8385\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 73s 58ms/step - loss: 0.2468 - accuracy: 0.9158 - val_loss: 0.3781 - val_accuracy: 0.8731\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 74s 58ms/step - loss: 0.0942 - accuracy: 0.9695 - val_loss: 0.4034 - val_accuracy: 0.8841\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.86      0.86       790\n",
      "           1       0.86      0.88      0.87       867\n",
      "           2       0.92      0.91      0.92       888\n",
      "\n",
      "    accuracy                           0.88      2545\n",
      "   macro avg       0.88      0.88      0.88      2545\n",
      "weighted avg       0.88      0.88      0.88      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.89 \n",
      "\n",
      "Test Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.98 \n",
      "\n",
      "Macro F1: 0.96 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.83 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.83 \n",
      "\n",
      "Macro F1: 0.8 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.73 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.83 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.67 \n",
      "\n",
      "Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.84 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 26\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 26\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 85s 66ms/step - loss: 0.6337 - accuracy: 0.7327 - val_loss: 0.4216 - val_accuracy: 0.8472\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 74s 58ms/step - loss: 0.3661 - accuracy: 0.8686 - val_loss: 0.3864 - val_accuracy: 0.8597\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 74s 58ms/step - loss: 0.2409 - accuracy: 0.9158 - val_loss: 0.3634 - val_accuracy: 0.8802\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 75s 59ms/step - loss: 0.0906 - accuracy: 0.9730 - val_loss: 0.4212 - val_accuracy: 0.8868\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.87      0.86       786\n",
      "           1       0.88      0.86      0.87       851\n",
      "           2       0.92      0.93      0.93       908\n",
      "\n",
      "    accuracy                           0.89      2545\n",
      "   macro avg       0.89      0.89      0.89      2545\n",
      "weighted avg       0.89      0.89      0.89      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.89 \n",
      "\n",
      "Test Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.81 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.82 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.78 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 26\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 10\n",
      "\t95percentile : 20\n",
      "\t99percentile : 27\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 84s 66ms/step - loss: 0.6224 - accuracy: 0.7361 - val_loss: 0.4333 - val_accuracy: 0.8432\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 74s 58ms/step - loss: 0.3633 - accuracy: 0.8678 - val_loss: 0.3865 - val_accuracy: 0.8633\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 73s 58ms/step - loss: 0.2422 - accuracy: 0.9180 - val_loss: 0.3698 - val_accuracy: 0.8802\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 73s 57ms/step - loss: 0.0962 - accuracy: 0.9684 - val_loss: 0.4086 - val_accuracy: 0.8888\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.85      0.87       827\n",
      "           1       0.87      0.87      0.87       822\n",
      "           2       0.91      0.94      0.92       896\n",
      "\n",
      "    accuracy                           0.89      2545\n",
      "   macro avg       0.89      0.89      0.89      2545\n",
      "weighted avg       0.89      0.89      0.89      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.89 \n",
      "\n",
      "Test Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.76 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.83 \n",
      "\n",
      "Macro F1: 0.78 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.77 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.81 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 10 fold cross validation, original dataset, without the question\n",
    "# 25% split for validation (each run is 67.5 / 22.5 / 10 for train / val / test)\n",
    "results_1 = tr_cls.mr_kfold_train_test(a_dataset[-1][1],0.25,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[   (   [   0.89,\n",
      "            [0.91, 0.86, 0.94, 0.83, 0.84, 0.82, 0.92, 0.91, 0.87, 0.93, 0.96],\n",
      "            [0.89, 0.81, 0.89, 0.9, 0.89, 0.87, 0.94, 1.0]],\n",
      "        [   0.89,\n",
      "            [0.85, 0.8, 0.9, 0.82, 0.83, 0.79, 0.85, 0.72, 0.84, 0.87, 0.94],\n",
      "            [0.62, 0.81, 0.89, 0.9, 0.89, 0.86, 0.93, 1.0]]),\n",
      "    (   [   0.89,\n",
      "            [0.85, 0.87, 0.9, 0.87, 0.86, 0.86, 0.94, 0.95, 0.84, 0.93, 0.88],\n",
      "            [0.78, 0.84, 0.89, 0.89, 0.9, 0.88, 0.84, 1.0]],\n",
      "        [   0.88,\n",
      "            [0.78, 0.82, 0.84, 0.87, 0.82, 0.85, 0.84, 0.64, 0.83, 0.89, 0.87],\n",
      "            [0.74, 0.85, 0.9, 0.89, 0.89, 0.88, 0.83, 1.0]]),\n",
      "    (   [   0.88,\n",
      "            [0.95, 0.84, 0.87, 0.86, 0.85, 0.85, 0.93, 0.91, 0.84, 0.94, 0.85],\n",
      "            [0.83, 0.85, 0.89, 0.9, 0.86, 0.85, 0.9, 1.0]],\n",
      "        [   0.88,\n",
      "            [0.94, 0.79, 0.78, 0.86, 0.81, 0.79, 0.82, 0.82, 0.81, 0.9, 0.85],\n",
      "            [0.79, 0.85, 0.89, 0.9, 0.85, 0.84, 0.89, 1.0]]),\n",
      "    (   [   0.88,\n",
      "            [0.86, 0.86, 0.85, 0.84, 0.88, 0.87, 0.87, 0.94, 0.83, 0.93, 0.93],\n",
      "            [0.82, 0.78, 0.89, 0.86, 0.89, 0.92, 0.87, 1.0]],\n",
      "        [   0.88,\n",
      "            [0.81, 0.81, 0.78, 0.77, 0.84, 0.85, 0.79, 0.75, 0.79, 0.89, 0.91],\n",
      "            [0.81, 0.78, 0.89, 0.86, 0.88, 0.92, 0.86, 1.0]]),\n",
      "    (   [   0.89,\n",
      "            [0.88, 0.81, 0.96, 0.91, 0.89, 0.85, 0.93, 0.91, 0.87, 0.93, 0.85],\n",
      "            [0.77, 0.8, 0.88, 0.91, 0.88, 0.9, 0.91, 1.0]],\n",
      "        [   0.89,\n",
      "            [0.83, 0.73, 0.95, 0.88, 0.85, 0.79, 0.82, 0.63, 0.83, 0.86, 0.79],\n",
      "            [0.77, 0.81, 0.88, 0.91, 0.88, 0.9, 0.9, 1.0]]),\n",
      "    (   [   0.89,\n",
      "            [0.83, 0.86, 0.92, 0.86, 0.95, 0.81, 0.96, 0.93, 0.87, 0.92, 0.88],\n",
      "            [0.92, 0.92, 0.91, 0.89, 0.87, 0.88, 0.89, 0.88]],\n",
      "        [   0.89,\n",
      "            [0.7, 0.78, 0.87, 0.79, 0.93, 0.79, 0.9, 0.9, 0.85, 0.89, 0.86],\n",
      "            [0.93, 0.92, 0.91, 0.89, 0.86, 0.87, 0.87, 0.89]]),\n",
      "    (   [   0.88,\n",
      "            [0.88, 0.87, 0.87, 0.85, 0.9, 0.84, 0.89, 0.95, 0.82, 0.93, 0.89],\n",
      "            [0.67, 0.88, 0.89, 0.86, 0.9, 0.88, 0.89, 1.0]],\n",
      "        [   0.88,\n",
      "            [0.87, 0.8, 0.82, 0.83, 0.87, 0.81, 0.84, 0.79, 0.78, 0.9, 0.85],\n",
      "            [0.53, 0.88, 0.89, 0.86, 0.9, 0.87, 0.88, 1.0]]),\n",
      "    (   [   0.89,\n",
      "            [0.89, 0.88, 0.98, 0.85, 0.89, 0.83, 0.93, 0.92, 0.83, 0.91, 0.86],\n",
      "            [0.67, 0.84, 0.87, 0.91, 0.9, 0.88, 0.91, 0.88]],\n",
      "        [   0.89,\n",
      "            [0.86, 0.84, 0.96, 0.83, 0.85, 0.8, 0.85, 0.73, 0.82, 0.86, 0.82],\n",
      "            [0.72, 0.84, 0.87, 0.91, 0.9, 0.87, 0.89, 0.89]]),\n",
      "    (   [   0.89,\n",
      "            [0.89, 0.85, 0.95, 0.9, 0.85, 0.82, 0.94, 0.92, 0.86, 0.93, 0.88],\n",
      "            [1.0, 0.85, 0.89, 0.89, 0.9, 0.86, 0.92, 1.0]],\n",
      "        [   0.89,\n",
      "            [0.88, 0.79, 0.89, 0.88, 0.81, 0.79, 0.84, 0.78, 0.84, 0.82, 0.87],\n",
      "            [1.0, 0.84, 0.9, 0.89, 0.9, 0.85, 0.92, 1.0]]),\n",
      "    (   [   0.89,\n",
      "            [0.88, 0.86, 0.91, 0.88, 0.91, 0.83, 0.93, 0.89, 0.86, 0.86, 0.91],\n",
      "            [0.86, 0.94, 0.88, 0.89, 0.85, 0.91, 0.89, 1.0]],\n",
      "        [   0.88,\n",
      "            [0.76, 0.82, 0.86, 0.86, 0.89, 0.78, 0.89, 0.77, 0.84, 0.81, 0.9],\n",
      "            [0.84, 0.94, 0.88, 0.88, 0.85, 0.91, 0.86, 1.0]])]\n"
     ]
    }
   ],
   "source": [
    "import pprint\n",
    "pp = pprint.PrettyPrinter(indent=4)\n",
    "pp.pprint(results_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc, F1, Avg-F1-Per-Q\n",
      "[   [0.89, 0.89, 0.89, 0.84],\n",
      "    [0.89, 0.88, 0.89, 0.82],\n",
      "    [0.88, 0.88, 0.88, 0.83],\n",
      "    [0.88, 0.88, 0.88, 0.82],\n",
      "    [0.89, 0.89, 0.89, 0.81],\n",
      "    [0.89, 0.89, 0.89, 0.84],\n",
      "    [0.88, 0.88, 0.88, 0.83],\n",
      "    [0.89, 0.89, 0.89, 0.84],\n",
      "    [0.89, 0.89, 0.89, 0.84],\n",
      "    [0.89, 0.88, 0.88, 0.83]]\n",
      "array([0.887, 0.885, 0.886, 0.83 ])\n",
      "\n",
      "Acc per question (avg over 10 runs)\n",
      "0.88 & 0.86 & 0.91 & 0.86 & 0.88 & 0.84 & 0.92 & 0.92 & 0.85 & 0.92 & 0.89\n",
      "\n",
      "F1 per question (avg over 10 runs)\n",
      "0.83 & 0.8 & 0.86 & 0.84 & 0.85 & 0.8 & 0.84 & 0.75 & 0.82 & 0.87 & 0.87\n"
     ]
    }
   ],
   "source": [
    "mr_proc_results(results_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change the model parameters to set max sent len to 35\n",
    "tr_cls.mr_set_model_vars(text_cols,[7,8,9,10,11,12,13,14],[0,1,2],35)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 17\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 18\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 95s 75ms/step - loss: 0.5766 - accuracy: 0.7717 - val_loss: 0.3499 - val_accuracy: 0.8715\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 84s 66ms/step - loss: 0.3273 - accuracy: 0.8864 - val_loss: 0.3607 - val_accuracy: 0.8680\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 85s 67ms/step - loss: 0.2232 - accuracy: 0.9280 - val_loss: 0.2844 - val_accuracy: 0.9143\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 85s 67ms/step - loss: 0.0942 - accuracy: 0.9701 - val_loss: 0.2850 - val_accuracy: 0.9218\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.88      0.90       748\n",
      "           1       0.89      0.93      0.91       874\n",
      "           2       0.95      0.95      0.95       923\n",
      "\n",
      "    accuracy                           0.92      2545\n",
      "   macro avg       0.92      0.92      0.92      2545\n",
      "weighted avg       0.92      0.92      0.92      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.91 \n",
      "\n",
      "Test Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.84 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.8 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.84 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Venelin\\AppData\\Roaming\\Python\\Python37\\site-packages\\ktrain\\text\\preprocessor.py:228: UserWarning: List or array of two texts supplied, so task being treated as text classification. If this is a sentence pair classification task, please cast to tuple.\n",
      "  warnings.warn('List or array of two texts supplied, so task being treated as text classification. ' +\\\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 17\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 17\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 93s 73ms/step - loss: 0.5786 - accuracy: 0.7641 - val_loss: 0.4044 - val_accuracy: 0.8558\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 84s 66ms/step - loss: 0.3292 - accuracy: 0.8833 - val_loss: 0.4341 - val_accuracy: 0.8546\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 84s 66ms/step - loss: 0.2122 - accuracy: 0.9274 - val_loss: 0.3091 - val_accuracy: 0.9010\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 84s 66ms/step - loss: 0.0843 - accuracy: 0.9754 - val_loss: 0.3382 - val_accuracy: 0.9081\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.89      0.89       791\n",
      "           1       0.90      0.89      0.90       885\n",
      "           2       0.93      0.94      0.93       869\n",
      "\n",
      "    accuracy                           0.91      2545\n",
      "   macro avg       0.91      0.91      0.91      2545\n",
      "weighted avg       0.91      0.91      0.91      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.92 \n",
      "\n",
      "Test Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.83 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.64 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 18\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 17\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 97s 76ms/step - loss: 0.5851 - accuracy: 0.7597 - val_loss: 0.3727 - val_accuracy: 0.8668\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 84s 66ms/step - loss: 0.3220 - accuracy: 0.8874 - val_loss: 0.3982 - val_accuracy: 0.8798\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 84s 66ms/step - loss: 0.2098 - accuracy: 0.9290 - val_loss: 0.2756 - val_accuracy: 0.9057\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 84s 66ms/step - loss: 0.0817 - accuracy: 0.9752 - val_loss: 0.3047 - val_accuracy: 0.9151\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.88      0.90       845\n",
      "           1       0.90      0.91      0.91       841\n",
      "           2       0.92      0.95      0.94       859\n",
      "\n",
      "    accuracy                           0.92      2545\n",
      "   macro avg       0.92      0.91      0.91      2545\n",
      "weighted avg       0.92      0.92      0.91      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.91 \n",
      "\n",
      "Test Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.81 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.83 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 17\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 18\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 94s 74ms/step - loss: 0.5865 - accuracy: 0.7666 - val_loss: 0.4281 - val_accuracy: 0.8519\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 85s 67ms/step - loss: 0.3348 - accuracy: 0.8826 - val_loss: 0.4446 - val_accuracy: 0.8487\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 85s 67ms/step - loss: 0.2228 - accuracy: 0.9277 - val_loss: 0.3268 - val_accuracy: 0.8959\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 86s 68ms/step - loss: 0.0976 - accuracy: 0.9696 - val_loss: 0.3373 - val_accuracy: 0.9065\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.87      0.89       807\n",
      "           1       0.89      0.90      0.89       847\n",
      "           2       0.92      0.94      0.93       891\n",
      "\n",
      "    accuracy                           0.91      2545\n",
      "   macro avg       0.91      0.91      0.91      2545\n",
      "weighted avg       0.91      0.91      0.91      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.91 \n",
      "\n",
      "Test Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.81 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.81 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.97 \n",
      "\n",
      "Macro F1: 0.95 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.96 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.82 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 17\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 17\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 97s 76ms/step - loss: 0.5791 - accuracy: 0.7700 - val_loss: 0.3560 - val_accuracy: 0.8727\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 86s 68ms/step - loss: 0.3265 - accuracy: 0.8857 - val_loss: 0.3229 - val_accuracy: 0.8829\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 85s 67ms/step - loss: 0.2245 - accuracy: 0.9251 - val_loss: 0.2937 - val_accuracy: 0.9037\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 86s 67ms/step - loss: 0.0885 - accuracy: 0.9726 - val_loss: 0.3086 - val_accuracy: 0.9120\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.88      0.89       799\n",
      "           1       0.90      0.90      0.90       867\n",
      "           2       0.93      0.95      0.94       879\n",
      "\n",
      "    accuracy                           0.91      2545\n",
      "   macro avg       0.91      0.91      0.91      2545\n",
      "weighted avg       0.91      0.91      0.91      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.91 \n",
      "\n",
      "Test Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.81 \n",
      "\n",
      "Macro F1: 0.71 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.98 \n",
      "\n",
      "Macro F1: 0.98 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.76 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.75 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.8 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.69 \n",
      "\n",
      "Macro F1: 0.68 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 17\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 17\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 96s 75ms/step - loss: 0.6066 - accuracy: 0.7536 - val_loss: 0.5302 - val_accuracy: 0.8134\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 85s 67ms/step - loss: 0.3397 - accuracy: 0.8803 - val_loss: 0.3410 - val_accuracy: 0.8845\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 85s 66ms/step - loss: 0.2344 - accuracy: 0.9230 - val_loss: 0.2948 - val_accuracy: 0.9018\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 85s 67ms/step - loss: 0.0888 - accuracy: 0.9716 - val_loss: 0.3130 - val_accuracy: 0.9139\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.88      0.90       809\n",
      "           1       0.90      0.91      0.91       876\n",
      "           2       0.93      0.95      0.94       860\n",
      "\n",
      "    accuracy                           0.91      2545\n",
      "   macro avg       0.91      0.91      0.91      2545\n",
      "weighted avg       0.91      0.91      0.91      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.92 \n",
      "\n",
      "Test Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.81 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.75 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.95 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.72 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.77 \n",
      "\n",
      "Macro F1: 0.75 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.86 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.95 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.75 \n",
      "\n",
      "Macro F1: 0.75 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 17\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 17\n",
      "\t95percentile : 28\n",
      "\t99percentile : 33\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 95s 75ms/step - loss: 0.5803 - accuracy: 0.7662 - val_loss: 0.3489 - val_accuracy: 0.8829\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 85s 67ms/step - loss: 0.3287 - accuracy: 0.8881 - val_loss: 0.3188 - val_accuracy: 0.8951\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 86s 67ms/step - loss: 0.2243 - accuracy: 0.9248 - val_loss: 0.2853 - val_accuracy: 0.9096\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 85s 67ms/step - loss: 0.0949 - accuracy: 0.9704 - val_loss: 0.2716 - val_accuracy: 0.9202\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.92      0.91       784\n",
      "           1       0.90      0.91      0.91       892\n",
      "           2       0.95      0.93      0.94       869\n",
      "\n",
      "    accuracy                           0.92      2545\n",
      "   macro avg       0.92      0.92      0.92      2545\n",
      "weighted avg       0.92      0.92      0.92      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.92 \n",
      "\n",
      "Test Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.79 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.96 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.96 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.83 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 17\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 18\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 96s 75ms/step - loss: 0.5921 - accuracy: 0.7526 - val_loss: 0.4280 - val_accuracy: 0.8244\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 85s 66ms/step - loss: 0.3305 - accuracy: 0.8842 - val_loss: 0.3812 - val_accuracy: 0.8872\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 85s 67ms/step - loss: 0.2232 - accuracy: 0.9248 - val_loss: 0.2576 - val_accuracy: 0.9132\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 86s 67ms/step - loss: 0.0824 - accuracy: 0.9746 - val_loss: 0.2837 - val_accuracy: 0.9222\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.90      0.90       753\n",
      "           1       0.91      0.92      0.92       868\n",
      "           2       0.94      0.95      0.95       924\n",
      "\n",
      "    accuracy                           0.92      2545\n",
      "   macro avg       0.92      0.92      0.92      2545\n",
      "weighted avg       0.92      0.92      0.92      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.91 \n",
      "\n",
      "Test Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.83 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.98 \n",
      "\n",
      "Macro F1: 0.96 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.85 \n",
      "\n",
      "Macro F1: 0.83 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.97 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.83 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 18\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 17\n",
      "\t95percentile : 27\n",
      "\t99percentile : 33\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 96s 75ms/step - loss: 0.5941 - accuracy: 0.7631 - val_loss: 0.3470 - val_accuracy: 0.8806\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 85s 67ms/step - loss: 0.3276 - accuracy: 0.8846 - val_loss: 0.3807 - val_accuracy: 0.8711\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 86s 68ms/step - loss: 0.2231 - accuracy: 0.9242 - val_loss: 0.3182 - val_accuracy: 0.8947\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 86s 67ms/step - loss: 0.0976 - accuracy: 0.9687 - val_loss: 0.3048 - val_accuracy: 0.9159\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.89      0.90       765\n",
      "           1       0.91      0.90      0.90       860\n",
      "           2       0.92      0.95      0.94       920\n",
      "\n",
      "    accuracy                           0.92      2545\n",
      "   macro avg       0.92      0.91      0.91      2545\n",
      "weighted avg       0.92      0.92      0.92      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.93 \n",
      "\n",
      "Test Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.83 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.96 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.87 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.98 \n",
      "\n",
      "Macro F1: 0.97 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.78 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.88 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.96 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.96 \n",
      "\n",
      "Macro F1: 0.97 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.93 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "preprocessing train...\n",
      "language: en\n",
      "train sequence lengths:\n",
      "\tmean : 17\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "preprocessing test...\n",
      "language: en\n",
      "test sequence lengths:\n",
      "\tmean : 17\n",
      "\t95percentile : 28\n",
      "\t99percentile : 34\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 8e-05...\n",
      "Train for 1273 steps, validate for 80 steps\n",
      "Epoch 1/4\n",
      "1273/1273 [==============================] - 95s 74ms/step - loss: 0.5941 - accuracy: 0.7593 - val_loss: 0.4738 - val_accuracy: 0.8338\n",
      "Epoch 2/4\n",
      "1273/1273 [==============================] - 84s 66ms/step - loss: 0.3449 - accuracy: 0.8806 - val_loss: 0.3747 - val_accuracy: 0.8750\n",
      "Epoch 3/4\n",
      "1273/1273 [==============================] - 86s 67ms/step - loss: 0.2168 - accuracy: 0.9238 - val_loss: 0.2936 - val_accuracy: 0.9088\n",
      "Epoch 4/4\n",
      "1273/1273 [==============================] - 87s 68ms/step - loss: 0.0822 - accuracy: 0.9734 - val_loss: 0.3105 - val_accuracy: 0.9163\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.89      0.89       768\n",
      "           1       0.90      0.92      0.91       898\n",
      "           2       0.95      0.94      0.94       879\n",
      "\n",
      "    accuracy                           0.92      2545\n",
      "   macro avg       0.92      0.92      0.92      2545\n",
      "weighted avg       0.92      0.92      0.92      2545\n",
      "\n",
      "Testing the model on the test set:\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.91 \n",
      "\n",
      "Test Macro F1: 0.91 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_1_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_2_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_3_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.89 \n",
      "\n",
      "Macro F1: 0.82 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_4_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_5_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.94 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Question with value SFQuestion_6_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.88 \n",
      "\n",
      "Macro F1: 0.85 \n",
      "\n",
      "Evaluating column Question with value SS_Brian_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.86 \n",
      "\n",
      "Evaluating column Question with value SS_Peabody_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93 \n",
      "\n",
      "Macro F1: 0.78 \n",
      "\n",
      "Evaluating column Question with value SS_Prisoner_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.91 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Question with value SS_Simon_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.84 \n",
      "\n",
      "Evaluating column Question with value SS_Burglar_Text\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.94 \n",
      "\n",
      "Evaluating column Age with value 7\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n",
      "Evaluating column Age with value 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.95 \n",
      "\n",
      "Macro F1: 0.95 \n",
      "\n",
      "Evaluating column Age with value 9\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Age with value 10\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 11\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9 \n",
      "\n",
      "Macro F1: 0.89 \n",
      "\n",
      "Evaluating column Age with value 12\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.92 \n",
      "\n",
      "Evaluating column Age with value 13\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.92 \n",
      "\n",
      "Macro F1: 0.9 \n",
      "\n",
      "Evaluating column Age with value 14\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0 \n",
      "\n",
      "Macro F1: 1.0 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "results_2 = tr_cls.mr_kfold_train_test(qa_dataset[-1][1],0.25,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[   (   [   0.91,\n",
      "            [0.93, 0.89, 0.95, 0.84, 0.89, 0.85, 0.92, 0.93, 0.91, 0.92, 0.95],\n",
      "            [1.0, 0.84, 0.89, 0.91, 0.93, 0.94, 0.93, 1.0]],\n",
      "        [   0.91,\n",
      "            [0.9, 0.82, 0.9, 0.79, 0.89, 0.8, 0.79, 0.79, 0.89, 0.85, 0.94],\n",
      "            [1.0, 0.84, 0.89, 0.91, 0.93, 0.94, 0.92, 1.0]]),\n",
      "    (   [   0.92,\n",
      "            [0.92, 0.87, 0.95, 0.92, 0.94, 0.85, 0.94, 0.94, 0.94, 0.93, 0.9],\n",
      "            [0.89, 0.86, 0.91, 0.92, 0.94, 0.9, 0.94, 1.0]],\n",
      "        [   0.92,\n",
      "            [0.92, 0.82, 0.94, 0.9, 0.93, 0.83, 0.84, 0.64, 0.94, 0.89, 0.89],\n",
      "            [0.82, 0.87, 0.92, 0.92, 0.93, 0.89, 0.94, 1.0]]),\n",
      "    (   [   0.91,\n",
      "            [0.94, 0.87, 0.94, 0.9, 0.88, 0.92, 0.93, 0.94, 0.88, 0.94, 0.91],\n",
      "            [1.0, 0.88, 0.92, 0.93, 0.9, 0.89, 0.92, 1.0]],\n",
      "        [   0.91,\n",
      "            [0.91, 0.81, 0.87, 0.88, 0.86, 0.89, 0.83, 0.9, 0.86, 0.89, 0.9],\n",
      "            [1.0, 0.88, 0.92, 0.92, 0.89, 0.88, 0.91, 1.0]]),\n",
      "    (   [   0.91,\n",
      "            [0.94, 0.88, 0.91, 0.89, 0.85, 0.9, 0.9, 0.94, 0.85, 0.97, 0.96],\n",
      "            [0.91, 0.82, 0.91, 0.91, 0.91, 0.92, 0.93, 1.0]],\n",
      "        [   0.91,\n",
      "            [0.93, 0.82, 0.86, 0.84, 0.81, 0.89, 0.82, 0.85, 0.81, 0.95, 0.94],\n",
      "            [0.93, 0.82, 0.91, 0.91, 0.91, 0.92, 0.92, 1.0]]),\n",
      "    (   [   0.91,\n",
      "            [0.92, 0.81, 0.98, 0.9, 0.94, 0.91, 0.93, 0.95, 0.91, 0.89, 0.89],\n",
      "            [0.69, 0.93, 0.9, 0.93, 0.9, 0.92, 0.92, 1.0]],\n",
      "        [   0.91,\n",
      "            [0.86, 0.71, 0.98, 0.84, 0.92, 0.87, 0.76, 0.86, 0.87, 0.75, 0.8],\n",
      "            [0.68, 0.94, 0.9, 0.93, 0.89, 0.91, 0.91, 1.0]]),\n",
      "    (   [   0.92,\n",
      "            [0.9, 0.94, 0.91, 0.85, 0.95, 0.95, 0.95, 0.93, 0.87, 0.94, 0.91],\n",
      "            [0.77, 0.86, 0.95, 0.92, 0.91, 0.9, 0.94, 0.75]],\n",
      "        [   0.92,\n",
      "            [0.81, 0.91, 0.87, 0.75, 0.95, 0.94, 0.88, 0.72, 0.86, 0.91, 0.89],\n",
      "            [0.75, 0.87, 0.95, 0.92, 0.91, 0.89, 0.93, 0.75]]),\n",
      "    (   [   0.92,\n",
      "            [0.95, 0.88, 0.92, 0.91, 0.9, 0.91, 0.96, 0.96, 0.9, 0.9, 0.93],\n",
      "            [0.83, 0.91, 0.93, 0.91, 0.93, 0.88, 0.95, 1.0]],\n",
      "        [   0.92,\n",
      "            [0.94, 0.79, 0.91, 0.89, 0.87, 0.9, 0.86, 0.87, 0.88, 0.84, 0.9],\n",
      "            [0.82, 0.92, 0.93, 0.91, 0.93, 0.87, 0.94, 1.0]]),\n",
      "    (   [   0.91,\n",
      "            [0.93, 0.87, 0.98, 0.87, 0.9, 0.85, 0.92, 0.97, 0.88, 0.95, 0.93],\n",
      "            [0.83, 0.93, 0.9, 0.93, 0.93, 0.89, 0.91, 0.88]],\n",
      "        [   0.91,\n",
      "            [0.92, 0.83, 0.96, 0.86, 0.89, 0.83, 0.82, 0.82, 0.87, 0.94, 0.92],\n",
      "            [0.84, 0.93, 0.9, 0.93, 0.93, 0.89, 0.9, 0.89]]),\n",
      "    (   [   0.93,\n",
      "            [0.94, 0.87, 0.96, 0.95, 0.91, 0.91, 0.98, 0.94, 0.9, 0.96, 0.96],\n",
      "            [1.0, 0.9, 0.93, 0.94, 0.94, 0.95, 0.93, 1.0]],\n",
      "        [   0.93,\n",
      "            [0.93, 0.83, 0.92, 0.94, 0.87, 0.9, 0.97, 0.78, 0.88, 0.9, 0.97],\n",
      "            [1.0, 0.89, 0.93, 0.94, 0.94, 0.94, 0.93, 1.0]]),\n",
      "    (   [   0.91,\n",
      "            [0.91, 0.87, 0.89, 0.91, 0.94, 0.88, 0.93, 0.93, 0.91, 0.9, 0.95],\n",
      "            [1.0, 0.95, 0.92, 0.9, 0.9, 0.92, 0.92, 1.0]],\n",
      "        [   0.91,\n",
      "            [0.85, 0.84, 0.82, 0.9, 0.94, 0.85, 0.86, 0.78, 0.9, 0.84, 0.94],\n",
      "            [1.0, 0.95, 0.92, 0.89, 0.89, 0.92, 0.9, 1.0]])]\n"
     ]
    }
   ],
   "source": [
    "pp.pprint(results_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc, F1, Avg-F1-Per-Q\n",
      "[   [0.91, 0.91, 0.91, 0.85],\n",
      "    [0.92, 0.92, 0.92, 0.87],\n",
      "    [0.91, 0.91, 0.91, 0.87],\n",
      "    [0.91, 0.91, 0.91, 0.87],\n",
      "    [0.91, 0.91, 0.91, 0.84],\n",
      "    [0.92, 0.92, 0.92, 0.86],\n",
      "    [0.92, 0.92, 0.92, 0.88],\n",
      "    [0.91, 0.91, 0.91, 0.88],\n",
      "    [0.93, 0.93, 0.93, 0.9],\n",
      "    [0.91, 0.91, 0.91, 0.87]]\n",
      "array([0.915, 0.915, 0.915, 0.869])\n",
      "\n",
      "Acc per question (avg over 10 runs)\n",
      "0.93 & 0.88 & 0.94 & 0.89 & 0.91 & 0.89 & 0.94 & 0.94 & 0.9 & 0.93 & 0.93\n",
      "\n",
      "F1 per question (avg over 10 runs)\n",
      "0.9 & 0.82 & 0.9 & 0.86 & 0.89 & 0.87 & 0.84 & 0.8 & 0.88 & 0.88 & 0.91\n"
     ]
    }
   ],
   "source": [
    "mr_proc_results(results_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}