TFIDF baseline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "\n",
    "%autoreload 2\n",
    "\n",
    "\n",
    "import warnings\n",
    "\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load tfidf model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "download from https://russiansuperglue.com/tasks/tf_idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import codecs\n",
    "import joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "vect = joblib.load(\"tfidf.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget https://russiansuperglue.com/tasks/download\n",
    "!unzip download\n",
    "!rm download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = Path(\"combined/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results = {\n",
    "    \"name\": [],\n",
    "    \"train\": [],\n",
    "    \"val\": [],\n",
    "    \"test\": []\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir = Path(\"baseline_submission\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir $output_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "\n",
    "def save_output(data, path):\n",
    "    with open(path, mode=\"w\") as file:\n",
    "        for line in sorted(data, key=lambda x: int(x.get(\"idx\"))):\n",
    "            line[\"idx\"] = int(line[\"idx\"])\n",
    "            file.write(f\"{json.dumps(line, ensure_ascii=False)}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PARus\n",
    "\n",
    "1. build text: \"{premise} {question} {choice1} {choice2}\"\n",
    "2. get tfidf\n",
    "3. fit logreg\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path = data_dir / \"PARus/train.jsonl\"\n",
    "val_path = data_dir / \"PARus/val.jsonl\"\n",
    "test_path = data_dir / \"PARus/test.jsonl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tfidf_baseline.PARus import eval_PARus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, PARus_scores = eval_PARus(train_path, val_path, test_path, vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7725, 0.44)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PARus_scores[\"train\"], PARus_scores[\"val\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results[\"name\"].append(\"PARus\")\n",
    "all_results[\"train\"].append(PARus_scores[\"train\"])\n",
    "all_results[\"val\"].append(PARus_scores[\"val\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_output(PARus_scores[\"test_pred\"], output_dir / \"PARus.jsonl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RCB\n",
    "1. build text: \"{premise} {hypothesis}\"\n",
    "2. get tfidf\n",
    "3. fit logreg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path = data_dir / \"RCB/train.jsonl\"\n",
    "val_path = data_dir / \"RCB/val.jsonl\"\n",
    "test_path = data_dir / \"RCB/test.jsonl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tfidf_baseline.RCB import eval_RCB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, RCB_scores = eval_RCB(train_path, val_path, test_path, vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7968036529680366, 0.5136363636363637)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RCB_scores[\"train\"], RCB_scores[\"val\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results[\"name\"].append(\"RCB\")\n",
    "all_results[\"train\"].append(RCB_scores[\"train\"])\n",
    "all_results[\"val\"].append(RCB_scores[\"val\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_output(RCB_scores[\"test_pred\"], output_dir / \"RCB.jsonl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DaNetQA\n",
    "1. build text: \"{question}\"\n",
    "2. get tfidf\n",
    "3. fit logreg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path = data_dir / \"DaNetQA/train.jsonl\"\n",
    "val_path = data_dir / \"DaNetQA/val.jsonl\"\n",
    "test_path = data_dir / \"DaNetQA/test.jsonl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tfidf_baseline.DaNetQA import eval_DaNetQA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, DaNetQA_scores = eval_DaNetQA(train_path, val_path, test_path, vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.8010291595197255, 0.5907429963459196)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DaNetQA_scores[\"train\"], DaNetQA_scores[\"val\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results[\"name\"].append(\"DaNetQA\")\n",
    "all_results[\"train\"].append(DaNetQA_scores[\"train\"])\n",
    "all_results[\"val\"].append(DaNetQA_scores[\"val\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_output(DaNetQA_scores[\"test_pred\"], output_dir / \"DaNetQA.jsonl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TERRa\n",
    "1. build text: \"{premise} {hypothesis}\"\n",
    "2. get tfidf\n",
    "3. fit logreg\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path = data_dir / \"TERRa/train.jsonl\"\n",
    "val_path = data_dir / \"TERRa/val.jsonl\"\n",
    "test_path = data_dir / \"TERRa/test.jsonl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tfidf_baseline.TERRa import eval_TERRa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, TERRa_scores = eval_TERRa(train_path, val_path, test_path, vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7152140672782875, 0.46579804560260585)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TERRa_scores[\"train\"], TERRa_scores[\"val\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results[\"name\"].append(\"TERRa\")\n",
    "all_results[\"train\"].append(TERRa_scores[\"train\"])\n",
    "all_results[\"val\"].append(TERRa_scores[\"val\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_output(TERRa_scores[\"test_pred\"], output_dir / \"TERRa.jsonl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RWSD\n",
    "1. build text: \"{premise} {span1} {span2}\"\n",
    "2. get tfidf\n",
    "3. fit logreg\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path = data_dir / \"RWSD/train.jsonl\"\n",
    "val_path = data_dir / \"RWSD/val.jsonl\"\n",
    "test_path = data_dir / \"RWSD/test.jsonl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tfidf_baseline.RWSD import eval_RWSD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, RWSD_scores = eval_RWSD(train_path, val_path, test_path, vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.5115511551155115, 0.553921568627451)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RWSD_scores[\"train\"], RWSD_scores[\"val\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results[\"name\"].append(\"RWSD\")\n",
    "all_results[\"train\"].append(RWSD_scores[\"train\"])\n",
    "all_results[\"val\"].append(RWSD_scores[\"val\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_output(RWSD_scores[\"test_pred\"], output_dir / \"RWSD.jsonl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RUSSE\n",
    "1. build text: \"{sentence1} {sentence2} {word}\"\n",
    "2. get tfidf\n",
    "3. fit logreg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path = data_dir / \"RUSSE/train.jsonl\"\n",
    "val_path = data_dir / \"RUSSE/val.jsonl\"\n",
    "test_path = data_dir / \"RUSSE/test.jsonl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tfidf_baseline.RUSSE import eval_RUSSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, RUSSE_scores = eval_RUSSE(train_path, val_path, test_path, vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7103552532123961, 0.6654908877131099)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RUSSE_scores[\"train\"], RUSSE_scores[\"val\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results[\"name\"].append(\"RUSSE\")\n",
    "all_results[\"train\"].append(RUSSE_scores[\"train\"])\n",
    "all_results[\"val\"].append(RUSSE_scores[\"val\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_output(RUSSE_scores[\"test_pred\"], output_dir / \"RUSSE.jsonl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LiDiRus\n",
    "1. build text: \"{sentence1} {sentence2}\"\n",
    "2. get tfidf\n",
    "3. fit logreg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path = data_dir / \"TERRa/train.jsonl\"\n",
    "val_path = data_dir / \"TERRa/val.jsonl\"\n",
    "test_path = data_dir / \"LiDiRus/LiDiRus.jsonl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tfidf_baseline.LiDiRus import eval_LiDiRus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, LiDiRus_scores = eval_LiDiRus(train_path, val_path, test_path, vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.4294719661883857, 0.05974021843803689)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LiDiRus_scores[\"train\"], LiDiRus_scores[\"test\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results[\"name\"].append(\"LiDiRus\")\n",
    "all_results[\"train\"].append(LiDiRus_scores[\"train\"])\n",
    "all_results[\"val\"].append(LiDiRus_scores[\"val\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_output(LiDiRus_scores[\"test_pred\"], output_dir / \"LiDiRus.jsonl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RuCoS\n",
    "1. build text of passage and queries\n",
    "2. get tfidf of passage and queries\n",
    "3. calculate cosins between passage and queries\n",
    "4. select best by cosin metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path = data_dir / \"RuCoS/train.jsonl\"\n",
    "val_path = data_dir / \"RuCoS/val.jsonl\"\n",
    "test_path = data_dir / \"RuCoS/test.jsonl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tfidf_baseline.RuCoS import eval_RuCoS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, RuCoS_scores = eval_RuCoS(train_path, val_path, test_path, vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((0.20824733699943207, 0.2263773335525391),\n",
       " (0.22964233865646033, 0.235315208305838))"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RuCoS_scores[\"train\"], RuCoS_scores[\"val\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results[\"name\"].append(\"RuCoS\")\n",
    "all_results[\"train\"].append(RuCoS_scores[\"train\"])\n",
    "all_results[\"val\"].append(RuCoS_scores[\"val\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_output(RuCoS_scores[\"test_pred\"], output_dir / \"RuCoS.jsonl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MuSeRC\n",
    "1. build text of passage and queries\n",
    "2. get tfidf of passage and queries\n",
    "3. calculate cosins between passage and queries\n",
    "4. select best 2 by cosin metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path = data_dir / \"MuSeRC/train.jsonl\"\n",
    "val_path = data_dir / \"MuSeRC/val.jsonl\"\n",
    "test_path = data_dir / \"MuSeRC/test.jsonl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tfidf_baseline.MuSeRC import eval_MuSeRC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, MuSeRC_scores = eval_MuSeRC(train_path, val_path, test_path, vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((0.2564722126337591, 0.5966356478167503),\n",
       " (0.2495274102079395, 0.5841053144807411))"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MuSeRC_scores[\"train\"], MuSeRC_scores[\"val\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results[\"name\"].append(\"MuSeRC\")\n",
    "all_results[\"train\"].append(MuSeRC_scores[\"train\"])\n",
    "all_results[\"val\"].append(MuSeRC_scores[\"val\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_output(MuSeRC_scores[\"test_pred\"], output_dir / \"MuSeRC.jsonl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Over All"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_results.pop(\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(all_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>train</th>\n",
       "      <th>val</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PARus</td>\n",
       "      <td>0.7725</td>\n",
       "      <td>0.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RCB</td>\n",
       "      <td>0.796804</td>\n",
       "      <td>0.513636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DaNetQA</td>\n",
       "      <td>0.801029</td>\n",
       "      <td>0.590743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TERRa</td>\n",
       "      <td>0.715214</td>\n",
       "      <td>0.465798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RWSD</td>\n",
       "      <td>0.511551</td>\n",
       "      <td>0.553922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RUSSE</td>\n",
       "      <td>0.710355</td>\n",
       "      <td>0.665491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>LiDiRus</td>\n",
       "      <td>0.429472</td>\n",
       "      <td>-0.0683523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>RuCoS</td>\n",
       "      <td>(0.20824733699943207, 0.2263773335525391)</td>\n",
       "      <td>(0.22964233865646033, 0.235315208305838)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>MuSeRC</td>\n",
       "      <td>(0.2564722126337591, 0.5966356478167503)</td>\n",
       "      <td>(0.2495274102079395, 0.5841053144807411)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      name                                      train  \\\n",
       "0    PARus                                     0.7725   \n",
       "1      RCB                                   0.796804   \n",
       "2  DaNetQA                                   0.801029   \n",
       "3    TERRa                                   0.715214   \n",
       "4     RWSD                                   0.511551   \n",
       "5    RUSSE                                   0.710355   \n",
       "6  LiDiRus                                   0.429472   \n",
       "7    RuCoS  (0.20824733699943207, 0.2263773335525391)   \n",
       "8   MuSeRC   (0.2564722126337591, 0.5966356478167503)   \n",
       "\n",
       "                                        val  \n",
       "0                                      0.44  \n",
       "1                                  0.513636  \n",
       "2                                  0.590743  \n",
       "3                                  0.465798  \n",
       "4                                  0.553922  \n",
       "5                                  0.665491  \n",
       "6                                -0.0683523  \n",
       "7  (0.22964233865646033, 0.235315208305838)  \n",
       "8  (0.2495274102079395, 0.5841053144807411)  "
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.to_csv(\"results.csv\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Make submission file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov : 2016-05-21\n",
      "p7zip Version 16.02 (locale=C.UTF-8,Utf16=on,HugeFiles=on,64 bits,96 CPUs Intel(R) Xeon(R) Platinum 8168 CPU @ 2.70GHz (50654),ASM,AES-NI)\n",
      "\n",
      "Scanning the drive:\n",
      "  0M Sca        1 folder, 9 files, 1403214 bytes (1371 KiB)\n",
      "\n",
      "Creating archive: baseline_submission.zip\n",
      "\n",
      "Items to compress: 10\n",
      "\n",
      "    \n",
      "Files read from disk: 9\n",
      "Archive size: 129050 bytes (127 KiB)\n",
      "Everything is Ok\n"
     ]
    }
   ],
   "source": [
    "!7z a \"baseline_submission.zip\" $output_dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Submit at https://russiansuperglue.com/login/start_submit/\n",
    "\n",
    "This submission should be scored with 0.434 total score"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}