last_ft_layer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "from collections import OrderedDict\n",
    "import logging\n",
    "import os\n",
    "import time\n",
    "from torch.multiprocessing import Pool, Process, set_start_method\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.model_selection import KFold, train_test_split\n",
    "from tqdm import tqdm\n",
    "\n",
    "from data_preprocess2 import split_overflow_table,fillup_table_w2i,remove_empty_cols,remove_empty_rows,tokenize_str\n",
    "from utils import Config, loadpkl, make_dirs, mp, flatten_1_deg,pool_fn,savepkl,print_tableIDs\n",
    "import models\n",
    "from dataset import T2VDataset\n",
    "from TREC_score import ndcg_pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_args():\n",
    "    parser = argparse.ArgumentParser()\n",
    "    parser.add_argument(\"--path\",\n",
    "                        help=\"path for the scores\")\n",
    "    parser.add_argument(\"-m\", \"--model_name\")\n",
    "    return parser.parse_args()\n",
    "\n",
    "class dotdict(dict):\n",
    "    \"\"\"dot.notation access to dictionary attributes\"\"\"\n",
    "    __getattr__ = dict.get\n",
    "    __setattr__ = dict.__setitem__\n",
    "    __delattr__ = dict.__delitem__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare the df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TREC_data_prep():\n",
    "    def __init__(self, config, vocab):\n",
    "        self.config = config\n",
    "        self.vocab = vocab\n",
    "\n",
    "    def convert2table(self, inp, typ):\n",
    "        if typ == 'table':\n",
    "            if np.array(inp).size != 0:\n",
    "                inp = remove_empty_cols(remove_empty_rows(inp))\n",
    "            if np.array(inp).size == 0:\n",
    "#             if len(inp) == 0:\n",
    "                inp = [[['<PAD>']]]\n",
    "        if typ == 'query':\n",
    "            inp = [[[j] for j in inp]]\n",
    "        return inp\n",
    "    \n",
    "    def shuffle_table(self,table):\n",
    "        c = max([len(row) for row in table])\n",
    "        r = len(table)\n",
    "        t = np.zeros((r,c)).tolist()\n",
    "        row_idx = list(range(r))\n",
    "        random.shuffle(row_idx)\n",
    "        col_idx = list(range(c))\n",
    "        random.shuffle(col_idx)\n",
    "        for i,row in enumerate(row_idx):\n",
    "            for j,col in enumerate(col_idx):\n",
    "                t[i][j]=table[row][col]\n",
    "        return t\n",
    "    \n",
    "    def prepare_data(self, df, typ):\n",
    "        pad_id = self.vocab.index('<PAD>')\n",
    "        if typ == 'table':\n",
    "            df[f\"{typ}_pad\"] = df[f\"{typ}_tkn\"].apply(lambda x: self.convert2table(eval(x),typ))\n",
    "#             df[f\"{typ}_pad\"] = df[f\"{typ}_pad\"].apply(lambda x: split_overflow_table(x))\n",
    "            df[f\"{typ}_pad\"] = df[f\"{typ}_pad\"].apply(lambda x: split_overflow_table(self.shuffle_table(x)))\n",
    "            for index, row in df.iterrows():\n",
    "                inp = row[f\"{typ}_pad\"]\n",
    "                inp = fillup_table_w2i(self.vocab,inp)\n",
    "                inp = [T2VDataset.pad_table(self.config['table_prep_params'], inp[i], pad_id) for i in range(len(inp))]\n",
    "                inp = np.array(inp)\n",
    "                print(index,inp.shape)\n",
    "                rows2add = [row] * inp.shape[0]\n",
    "                for i in range(inp.shape[0]):\n",
    "                    rows2add[i][f\"{typ}_pad\"] = inp[i]\n",
    "                df = df.drop(index)\n",
    "                df = pd.concat([df, pd.DataFrame(rows2add)])\n",
    "        elif typ == 'query':\n",
    "            tables = {t:t for t in list(set(df[f\"{typ}_tkn\"]))}\n",
    "            print(tables)\n",
    "            for i in tables.keys():\n",
    "                t = self.convert2table(eval(tables[i]),typ)\n",
    "                t = fillup_table_w2i(self.vocab,[t])\n",
    "                tables[i] = np.array(T2VDataset.pad_table(config['table_prep_params'], t[0], pad_id))\n",
    "            df[f\"{typ}_pad\"] = df[f\"{typ}_tkn\"].apply(lambda x: tables[x])\n",
    "        print(df.shape)\n",
    "        return df\n",
    "    \n",
    "    def pipeline(self, baseline_f):\n",
    "        baseline_f = self.prepare_data(baseline_f, 'table')\n",
    "        baseline_f = self.prepare_data(baseline_f, 'query')\n",
    "        return baseline_f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# args = get_args()\n",
    "args = dotdict({\"path\":'output/6_29_13_36_22', 'model_name':'model_40.pt'})\n",
    "\n",
    "config = Config()\n",
    "config.load(os.path.join(args.path, 'config.toml'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = loadpkl(config['input_files']['vocab_path'])\n",
    "device = torch.device(f\"cuda:{config['gpu']}\")\n",
    "\n",
    "# vocab = loadpkl('/home/vibhav_student/2Dcnn/data/w_all_data/vocab_5-15_unk_qfix.pkl')\n",
    "# device = torch.device('cuda:3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 (1, 15, 5, 1)\n",
      "1 (1, 15, 5, 1)\n",
      "2 (1, 15, 5, 1)\n",
      "3 (1, 15, 5, 1)\n",
      "4 (1, 15, 5, 1)\n",
      "5 (1, 15, 5, 1)\n",
      "6 (1, 15, 5, 1)\n",
      "7 (5, 15, 5, 1)\n",
      "8 (1, 15, 5, 1)\n",
      "9 (1, 15, 5, 1)\n",
      "10 (1, 15, 5, 1)\n",
      "11 (2, 15, 5, 1)\n",
      "12 (2, 15, 5, 1)\n",
      "13 (1, 15, 5, 1)\n",
      "14 (1, 15, 5, 1)\n",
      "15 (2, 15, 5, 1)\n",
      "16 (1, 15, 5, 1)\n",
      "17 (1, 15, 5, 1)\n",
      "18 (1, 15, 5, 1)\n",
      "19 (2, 15, 5, 1)\n",
      "20 (1, 15, 5, 1)\n",
      "21 (2, 15, 5, 1)\n",
      "22 (1, 15, 5, 1)\n",
      "23 (1, 15, 5, 1)\n",
      "24 (1, 15, 5, 1)\n",
      "25 (1, 15, 5, 1)\n",
      "26 (1, 15, 5, 1)\n",
      "27 (1, 15, 5, 1)\n",
      "28 (1, 15, 5, 1)\n",
      "29 (1, 15, 5, 1)\n",
      "30 (1, 15, 5, 1)\n",
      "31 (1, 15, 5, 1)\n",
      "32 (1, 15, 5, 1)\n",
      "33 (1, 15, 5, 1)\n",
      "34 (1, 15, 5, 1)\n",
      "35 (7, 15, 5, 1)\n",
      "36 (1, 15, 5, 1)\n",
      "37 (3, 15, 5, 1)\n",
      "38 (2, 15, 5, 1)\n",
      "39 (1, 15, 5, 1)\n",
      "40 (5, 15, 5, 1)\n",
      "41 (1, 15, 5, 1)\n",
      "42 (2, 15, 5, 1)\n",
      "43 (3, 15, 5, 1)\n",
      "44 (1, 15, 5, 1)\n",
      "45 (1, 15, 5, 1)\n",
      "46 (1, 15, 5, 1)\n",
      "47 (4, 15, 5, 1)\n",
      "48 (1, 15, 5, 1)\n",
      "49 (1, 15, 5, 1)\n",
      "50 (1, 15, 5, 1)\n",
      "51 (1, 15, 5, 1)\n",
      "52 (1, 15, 5, 1)\n",
      "53 (16, 15, 5, 1)\n",
      "54 (1, 15, 5, 1)\n",
      "55 (2, 15, 5, 1)\n",
      "56 (1, 15, 5, 1)\n",
      "57 (1, 15, 5, 1)\n",
      "58 (1, 15, 5, 1)\n",
      "59 (1, 15, 5, 1)\n",
      "60 (1, 15, 5, 1)\n",
      "61 (1, 15, 5, 1)\n",
      "62 (1, 15, 5, 1)\n",
      "63 (1, 15, 5, 1)\n",
      "64 (1, 15, 5, 1)\n",
      "65 (1, 15, 5, 1)\n",
      "66 (1, 15, 5, 1)\n",
      "67 (1, 15, 5, 1)\n",
      "68 (1, 15, 5, 1)\n",
      "69 (1, 15, 5, 1)\n",
      "70 (1, 15, 5, 1)\n",
      "71 (1, 15, 5, 1)\n",
      "72 (1, 15, 5, 1)\n",
      "73 (1, 15, 5, 1)\n",
      "74 (1, 15, 5, 1)\n",
      "75 (1, 15, 5, 1)\n",
      "76 (1, 15, 5, 1)\n",
      "77 (1, 15, 5, 1)\n",
      "78 (1, 15, 5, 1)\n",
      "79 (4, 15, 5, 1)\n",
      "80 (1, 15, 5, 1)\n",
      "81 (1, 15, 5, 1)\n",
      "82 (1, 15, 5, 1)\n",
      "83 (1, 15, 5, 1)\n",
      "84 (1, 15, 5, 1)\n",
      "85 (1, 15, 5, 1)\n",
      "86 (1, 15, 5, 1)\n",
      "87 (1, 15, 5, 1)\n",
      "88 (1, 15, 5, 1)\n",
      "89 (1, 15, 5, 1)\n",
      "90 (1, 15, 5, 1)\n",
      "91 (1, 15, 5, 1)\n",
      "92 (1, 15, 5, 1)\n",
      "93 (1, 15, 5, 1)\n",
      "94 (1, 15, 5, 1)\n",
      "95 (1, 15, 5, 1)\n",
      "96 (1, 15, 5, 1)\n",
      "97 (1, 15, 5, 1)\n",
      "98 (2, 15, 5, 1)\n",
      "99 (1, 15, 5, 1)\n",
      "100 (1, 15, 5, 1)\n",
      "101 (1, 15, 5, 1)\n",
      "102 (1, 15, 5, 1)\n",
      "103 (1, 15, 5, 1)\n",
      "104 (1, 15, 5, 1)\n",
      "105 (1, 15, 5, 1)\n",
      "106 (1, 15, 5, 1)\n",
      "107 (1, 15, 5, 1)\n",
      "108 (1, 15, 5, 1)\n",
      "109 (1, 15, 5, 1)\n",
      "110 (2, 15, 5, 1)\n",
      "111 (1, 15, 5, 1)\n",
      "112 (1, 15, 5, 1)\n",
      "113 (1, 15, 5, 1)\n",
      "114 (1, 15, 5, 1)\n",
      "115 (1, 15, 5, 1)\n",
      "116 (1, 15, 5, 1)\n",
      "117 (1, 15, 5, 1)\n",
      "118 (1, 15, 5, 1)\n",
      "119 (1, 15, 5, 1)\n",
      "120 (1, 15, 5, 1)\n",
      "121 (1, 15, 5, 1)\n",
      "122 (1, 15, 5, 1)\n",
      "123 (1, 15, 5, 1)\n",
      "124 (1, 15, 5, 1)\n",
      "125 (1, 15, 5, 1)\n",
      "126 (1, 15, 5, 1)\n",
      "127 (2, 15, 5, 1)\n",
      "128 (1, 15, 5, 1)\n",
      "129 (1, 15, 5, 1)\n",
      "130 (1, 15, 5, 1)\n",
      "131 (1, 15, 5, 1)\n",
      "132 (1, 15, 5, 1)\n",
      "133 (2, 15, 5, 1)\n",
      "134 (1, 15, 5, 1)\n",
      "135 (4, 15, 5, 1)\n",
      "136 (1, 15, 5, 1)\n",
      "137 (1, 15, 5, 1)\n",
      "138 (2, 15, 5, 1)\n",
      "139 (2, 15, 5, 1)\n",
      "140 (1, 15, 5, 1)\n",
      "141 (1, 15, 5, 1)\n",
      "142 (1, 15, 5, 1)\n",
      "143 (1, 15, 5, 1)\n",
      "144 (1, 15, 5, 1)\n",
      "145 (1, 15, 5, 1)\n",
      "146 (1, 15, 5, 1)\n",
      "147 (2, 15, 5, 1)\n",
      "148 (2, 15, 5, 1)\n",
      "149 (5, 15, 5, 1)\n",
      "150 (17, 15, 5, 1)\n",
      "151 (3, 15, 5, 1)\n",
      "152 (2, 15, 5, 1)\n",
      "153 (1, 15, 5, 1)\n",
      "154 (2, 15, 5, 1)\n",
      "155 (2, 15, 5, 1)\n",
      "156 (2, 15, 5, 1)\n",
      "157 (1, 15, 5, 1)\n",
      "158 (1, 15, 5, 1)\n",
      "159 (1, 15, 5, 1)\n",
      "160 (1, 15, 5, 1)\n",
      "161 (1, 15, 5, 1)\n",
      "162 (1, 15, 5, 1)\n",
      "163 (2, 15, 5, 1)\n",
      "164 (2, 15, 5, 1)\n",
      "165 (1, 15, 5, 1)\n",
      "166 (1, 15, 5, 1)\n",
      "167 (1, 15, 5, 1)\n",
      "168 (1, 15, 5, 1)\n",
      "169 (1, 15, 5, 1)\n",
      "170 (1, 15, 5, 1)\n",
      "171 (1, 15, 5, 1)\n",
      "172 (1, 15, 5, 1)\n",
      "173 (1, 15, 5, 1)\n",
      "174 (1, 15, 5, 1)\n",
      "175 (1, 15, 5, 1)\n",
      "176 (1, 15, 5, 1)\n",
      "177 (1, 15, 5, 1)\n",
      "178 (1, 15, 5, 1)\n",
      "179 (1, 15, 5, 1)\n",
      "180 (1, 15, 5, 1)\n",
      "181 (1, 15, 5, 1)\n",
      "182 (1, 15, 5, 1)\n",
      "183 (1, 15, 5, 1)\n",
      "184 (1, 15, 5, 1)\n",
      "185 (1, 15, 5, 1)\n",
      "186 (1, 15, 5, 1)\n",
      "187 (1, 15, 5, 1)\n",
      "188 (1, 15, 5, 1)\n",
      "189 (1, 15, 5, 1)\n",
      "190 (1, 15, 5, 1)\n",
      "191 (1, 15, 5, 1)\n",
      "192 (1, 15, 5, 1)\n",
      "193 (1, 15, 5, 1)\n",
      "194 (2, 15, 5, 1)\n",
      "195 (1, 15, 5, 1)\n",
      "196 (3, 15, 5, 1)\n",
      "197 (1, 15, 5, 1)\n",
      "198 (1, 15, 5, 1)\n",
      "199 (2, 15, 5, 1)\n",
      "200 (2, 15, 5, 1)\n",
      "201 (1, 15, 5, 1)\n",
      "202 (1, 15, 5, 1)\n",
      "203 (6, 15, 5, 1)\n",
      "204 (2, 15, 5, 1)\n",
      "205 (7, 15, 5, 1)\n",
      "206 (1, 15, 5, 1)\n",
      "207 (1, 15, 5, 1)\n",
      "208 (1, 15, 5, 1)\n",
      "209 (2, 15, 5, 1)\n",
      "210 (1, 15, 5, 1)\n",
      "211 (1, 15, 5, 1)\n",
      "212 (1, 15, 5, 1)\n",
      "213 (1, 15, 5, 1)\n",
      "214 (1, 15, 5, 1)\n",
      "215 (1, 15, 5, 1)\n",
      "216 (1, 15, 5, 1)\n",
      "217 (1, 15, 5, 1)\n",
      "218 (2, 15, 5, 1)\n",
      "219 (1, 15, 5, 1)\n",
      "220 (1, 15, 5, 1)\n",
      "221 (1, 15, 5, 1)\n",
      "222 (1, 15, 5, 1)\n",
      "223 (1, 15, 5, 1)\n",
      "224 (1, 15, 5, 1)\n",
      "225 (1, 15, 5, 1)\n",
      "226 (1, 15, 5, 1)\n",
      "227 (1, 15, 5, 1)\n",
      "228 (3, 15, 5, 1)\n",
      "229 (1, 15, 5, 1)\n",
      "230 (1, 15, 5, 1)\n",
      "231 (2, 15, 5, 1)\n",
      "232 (2, 15, 5, 1)\n",
      "233 (2, 15, 5, 1)\n",
      "234 (2, 15, 5, 1)\n",
      "235 (2, 15, 5, 1)\n",
      "236 (1, 15, 5, 1)\n",
      "237 (2, 15, 5, 1)\n",
      "238 (5, 15, 5, 1)\n",
      "239 (7, 15, 5, 1)\n",
      "240 (1, 15, 5, 1)\n",
      "241 (2, 15, 5, 1)\n",
      "242 (2, 15, 5, 1)\n",
      "243 (1, 15, 5, 1)\n",
      "244 (1, 15, 5, 1)\n",
      "245 (1, 15, 5, 1)\n",
      "246 (1, 15, 5, 1)\n",
      "247 (1, 15, 5, 1)\n",
      "248 (1, 15, 5, 1)\n",
      "249 (1, 15, 5, 1)\n",
      "250 (1, 15, 5, 1)\n",
      "251 (1, 15, 5, 1)\n",
      "252 (1, 15, 5, 1)\n",
      "253 (1, 15, 5, 1)\n",
      "254 (1, 15, 5, 1)\n",
      "255 (1, 15, 5, 1)\n",
      "256 (1, 15, 5, 1)\n",
      "257 (1, 15, 5, 1)\n",
      "258 (10, 15, 5, 1)\n",
      "259 (1, 15, 5, 1)\n",
      "260 (1, 15, 5, 1)\n",
      "261 (1, 15, 5, 1)\n",
      "262 (1, 15, 5, 1)\n",
      "263 (1, 15, 5, 1)\n",
      "264 (1, 15, 5, 1)\n",
      "265 (1, 15, 5, 1)\n",
      "266 (1, 15, 5, 1)\n",
      "267 (1, 15, 5, 1)\n",
      "268 (1, 15, 5, 1)\n",
      "269 (1, 15, 5, 1)\n",
      "270 (2, 15, 5, 1)\n",
      "271 (1, 15, 5, 1)\n",
      "272 (1, 15, 5, 1)\n",
      "273 (1, 15, 5, 1)\n",
      "274 (4, 15, 5, 1)\n",
      "275 (1, 15, 5, 1)\n",
      "276 (2, 15, 5, 1)\n",
      "277 (4, 15, 5, 1)\n",
      "278 (4, 15, 5, 1)\n",
      "279 (4, 15, 5, 1)\n",
      "280 (1, 15, 5, 1)\n",
      "281 (2, 15, 5, 1)\n",
      "282 (3, 15, 5, 1)\n",
      "283 (3, 15, 5, 1)\n",
      "284 (3, 15, 5, 1)\n",
      "285 (3, 15, 5, 1)\n",
      "286 (1, 15, 5, 1)\n",
      "287 (1, 15, 5, 1)\n",
      "288 (2, 15, 5, 1)\n",
      "289 (2, 15, 5, 1)\n",
      "290 (1, 15, 5, 1)\n",
      "291 (3, 15, 5, 1)\n",
      "292 (1, 15, 5, 1)\n",
      "293 (1, 15, 5, 1)\n",
      "294 (1, 15, 5, 1)\n",
      "295 (1, 15, 5, 1)\n",
      "296 (1, 15, 5, 1)\n",
      "297 (1, 15, 5, 1)\n",
      "298 (3, 15, 5, 1)\n",
      "299 (1, 15, 5, 1)\n",
      "300 (2, 15, 5, 1)\n",
      "301 (1, 15, 5, 1)\n",
      "302 (3, 15, 5, 1)\n",
      "303 (1, 15, 5, 1)\n",
      "304 (1, 15, 5, 1)\n",
      "305 (1, 15, 5, 1)\n",
      "306 (2, 15, 5, 1)\n",
      "307 (2, 15, 5, 1)\n",
      "308 (2, 15, 5, 1)\n",
      "309 (1, 15, 5, 1)\n",
      "310 (1, 15, 5, 1)\n",
      "311 (2, 15, 5, 1)\n",
      "312 (4, 15, 5, 1)\n",
      "313 (1, 15, 5, 1)\n",
      "314 (1, 15, 5, 1)\n",
      "315 (1, 15, 5, 1)\n",
      "316 (1, 15, 5, 1)\n",
      "317 (1, 15, 5, 1)\n",
      "318 (1, 15, 5, 1)\n",
      "319 (1, 15, 5, 1)\n",
      "320 (1, 15, 5, 1)\n",
      "321 (1, 15, 5, 1)\n",
      "322 (1, 15, 5, 1)\n",
      "323 (2, 15, 5, 1)\n",
      "324 (1, 15, 5, 1)\n",
      "325 (2, 15, 5, 1)\n",
      "326 (6, 15, 5, 1)\n",
      "327 (1, 15, 5, 1)\n",
      "328 (1, 15, 5, 1)\n",
      "329 (2, 15, 5, 1)\n",
      "330 (2, 15, 5, 1)\n",
      "331 (2, 15, 5, 1)\n",
      "332 (1, 15, 5, 1)\n",
      "333 (1, 15, 5, 1)\n",
      "334 (1, 15, 5, 1)\n",
      "335 (2, 15, 5, 1)\n",
      "336 (2, 15, 5, 1)\n",
      "337 (1, 15, 5, 1)\n",
      "338 (1, 15, 5, 1)\n",
      "339 (1, 15, 5, 1)\n",
      "340 (1, 15, 5, 1)\n",
      "341 (4, 15, 5, 1)\n",
      "342 (1, 15, 5, 1)\n",
      "343 (1, 15, 5, 1)\n",
      "344 (1, 15, 5, 1)\n",
      "345 (1, 15, 5, 1)\n",
      "346 (1, 15, 5, 1)\n",
      "347 (1, 15, 5, 1)\n",
      "348 (1, 15, 5, 1)\n",
      "349 (1, 15, 5, 1)\n",
      "350 (1, 15, 5, 1)\n",
      "351 (1, 15, 5, 1)\n",
      "352 (1, 15, 5, 1)\n",
      "353 (1, 15, 5, 1)\n",
      "354 (1, 15, 5, 1)\n",
      "355 (10, 15, 5, 1)\n",
      "356 (1, 15, 5, 1)\n",
      "357 (1, 15, 5, 1)\n",
      "358 (1, 15, 5, 1)\n",
      "359 (1, 15, 5, 1)\n",
      "360 (2, 15, 5, 1)\n",
      "361 (1, 15, 5, 1)\n",
      "362 (1, 15, 5, 1)\n",
      "363 (1, 15, 5, 1)\n",
      "364 (2, 15, 5, 1)\n",
      "365 (1, 15, 5, 1)\n",
      "366 (4, 15, 5, 1)\n",
      "367 (1, 15, 5, 1)\n",
      "368 (1, 15, 5, 1)\n",
      "369 (1, 15, 5, 1)\n",
      "370 (1, 15, 5, 1)\n",
      "371 (1, 15, 5, 1)\n",
      "372 (1, 15, 5, 1)\n",
      "373 (1, 15, 5, 1)\n",
      "374 (1, 15, 5, 1)\n",
      "375 (1, 15, 5, 1)\n",
      "376 (1, 15, 5, 1)\n",
      "377 (1, 15, 5, 1)\n",
      "378 (1, 15, 5, 1)\n",
      "379 (1, 15, 5, 1)\n",
      "380 (2, 15, 5, 1)\n",
      "381 (2, 15, 5, 1)\n",
      "382 (4, 15, 5, 1)\n",
      "383 (1, 15, 5, 1)\n",
      "384 (1, 15, 5, 1)\n",
      "385 (2, 15, 5, 1)\n",
      "386 (2, 15, 5, 1)\n",
      "387 (2, 15, 5, 1)\n",
      "388 (8, 15, 5, 1)\n",
      "389 (1, 15, 5, 1)\n",
      "390 (1, 15, 5, 1)\n",
      "391 (1, 15, 5, 1)\n",
      "392 (1, 15, 5, 1)\n",
      "393 (3, 15, 5, 1)\n",
      "394 (1, 15, 5, 1)\n",
      "395 (3, 15, 5, 1)\n",
      "396 (1, 15, 5, 1)\n",
      "397 (1, 15, 5, 1)\n",
      "398 (1, 15, 5, 1)\n",
      "399 (2, 15, 5, 1)\n",
      "400 (2, 15, 5, 1)\n",
      "401 (1, 15, 5, 1)\n",
      "402 (1, 15, 5, 1)\n",
      "403 (2, 15, 5, 1)\n",
      "404 (1, 15, 5, 1)\n",
      "405 (2, 15, 5, 1)\n",
      "406 (1, 15, 5, 1)\n",
      "407 (17, 15, 5, 1)\n",
      "408 (1, 15, 5, 1)\n",
      "409 (2, 15, 5, 1)\n",
      "410 (1, 15, 5, 1)\n",
      "411 (1, 15, 5, 1)\n",
      "412 (3, 15, 5, 1)\n",
      "413 (1, 15, 5, 1)\n",
      "414 (1, 15, 5, 1)\n",
      "415 (1, 15, 5, 1)\n",
      "416 (1, 15, 5, 1)\n",
      "417 (1, 15, 5, 1)\n",
      "418 (1, 15, 5, 1)\n",
      "419 (1, 15, 5, 1)\n",
      "420 (1, 15, 5, 1)\n",
      "421 (1, 15, 5, 1)\n",
      "422 (1, 15, 5, 1)\n",
      "423 (1, 15, 5, 1)\n",
      "424 (1, 15, 5, 1)\n",
      "425 (1, 15, 5, 1)\n",
      "426 (1, 15, 5, 1)\n",
      "427 (1, 15, 5, 1)\n",
      "428 (1, 15, 5, 1)\n",
      "429 (8, 15, 5, 1)\n",
      "430 (2, 15, 5, 1)\n",
      "431 (4, 15, 5, 1)\n",
      "432 (1, 15, 5, 1)\n",
      "433 (1, 15, 5, 1)\n",
      "434 (1, 15, 5, 1)\n",
      "435 (1, 15, 5, 1)\n",
      "436 (1, 15, 5, 1)\n",
      "437 (1, 15, 5, 1)\n",
      "438 (1, 15, 5, 1)\n",
      "439 (2, 15, 5, 1)\n",
      "440 (1, 15, 5, 1)\n",
      "441 (1, 15, 5, 1)\n",
      "442 (1, 15, 5, 1)\n",
      "443 (1, 15, 5, 1)\n",
      "444 (1, 15, 5, 1)\n",
      "445 (1, 15, 5, 1)\n",
      "446 (2, 15, 5, 1)\n",
      "447 (1, 15, 5, 1)\n",
      "448 (6, 15, 5, 1)\n",
      "449 (2, 15, 5, 1)\n",
      "450 (1, 15, 5, 1)\n",
      "451 (1, 15, 5, 1)\n",
      "452 (1, 15, 5, 1)\n",
      "453 (2, 15, 5, 1)\n",
      "454 (2, 15, 5, 1)\n",
      "455 (2, 15, 5, 1)\n",
      "456 (4, 15, 5, 1)\n",
      "457 (1, 15, 5, 1)\n",
      "458 (1, 15, 5, 1)\n",
      "459 (2, 15, 5, 1)\n",
      "460 (1, 15, 5, 1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "461 (1, 15, 5, 1)\n",
      "462 (2, 15, 5, 1)\n",
      "463 (1, 15, 5, 1)\n",
      "464 (4, 15, 5, 1)\n",
      "465 (1, 15, 5, 1)\n",
      "466 (1, 15, 5, 1)\n",
      "467 (1, 15, 5, 1)\n",
      "468 (2, 15, 5, 1)\n",
      "469 (2, 15, 5, 1)\n",
      "470 (4, 15, 5, 1)\n",
      "471 (4, 15, 5, 1)\n",
      "472 (1, 15, 5, 1)\n",
      "473 (1, 15, 5, 1)\n",
      "474 (7, 15, 5, 1)\n",
      "475 (1, 15, 5, 1)\n",
      "476 (3, 15, 5, 1)\n",
      "477 (24, 15, 5, 1)\n",
      "478 (24, 15, 5, 1)\n",
      "479 (13, 15, 5, 1)\n",
      "480 (2, 15, 5, 1)\n",
      "481 (1, 15, 5, 1)\n",
      "482 (1, 15, 5, 1)\n",
      "483 (1, 15, 5, 1)\n",
      "484 (2, 15, 5, 1)\n",
      "485 (1, 15, 5, 1)\n",
      "486 (1, 15, 5, 1)\n",
      "487 (2, 15, 5, 1)\n",
      "488 (1, 15, 5, 1)\n",
      "489 (1, 15, 5, 1)\n",
      "490 (1, 15, 5, 1)\n",
      "491 (2, 15, 5, 1)\n",
      "492 (3, 15, 5, 1)\n",
      "493 (2, 15, 5, 1)\n",
      "494 (3, 15, 5, 1)\n",
      "495 (1, 15, 5, 1)\n",
      "496 (2, 15, 5, 1)\n",
      "497 (1, 15, 5, 1)\n",
      "498 (1, 15, 5, 1)\n",
      "499 (1, 15, 5, 1)\n",
      "500 (3, 15, 5, 1)\n",
      "501 (3, 15, 5, 1)\n",
      "502 (1, 15, 5, 1)\n",
      "503 (3, 15, 5, 1)\n",
      "504 (2, 15, 5, 1)\n",
      "505 (1, 15, 5, 1)\n",
      "506 (1, 15, 5, 1)\n",
      "507 (3, 15, 5, 1)\n",
      "508 (1, 15, 5, 1)\n",
      "509 (3, 15, 5, 1)\n",
      "510 (4, 15, 5, 1)\n",
      "511 (1, 15, 5, 1)\n",
      "512 (8, 15, 5, 1)\n",
      "513 (1, 15, 5, 1)\n",
      "514 (1, 15, 5, 1)\n",
      "515 (8, 15, 5, 1)\n",
      "516 (2, 15, 5, 1)\n",
      "517 (1, 15, 5, 1)\n",
      "518 (12, 15, 5, 1)\n",
      "519 (1, 15, 5, 1)\n",
      "520 (1, 15, 5, 1)\n",
      "521 (1, 15, 5, 1)\n",
      "522 (1, 15, 5, 1)\n",
      "523 (1, 15, 5, 1)\n",
      "524 (4, 15, 5, 1)\n",
      "525 (1, 15, 5, 1)\n",
      "526 (1, 15, 5, 1)\n",
      "527 (1, 15, 5, 1)\n",
      "528 (1, 15, 5, 1)\n",
      "529 (1, 15, 5, 1)\n",
      "530 (1, 15, 5, 1)\n",
      "531 (1, 15, 5, 1)\n",
      "532 (1, 15, 5, 1)\n",
      "533 (2, 15, 5, 1)\n",
      "534 (1, 15, 5, 1)\n",
      "535 (1, 15, 5, 1)\n",
      "536 (1, 15, 5, 1)\n",
      "537 (1, 15, 5, 1)\n",
      "538 (1, 15, 5, 1)\n",
      "539 (1, 15, 5, 1)\n",
      "540 (1, 15, 5, 1)\n",
      "541 (1, 15, 5, 1)\n",
      "542 (1, 15, 5, 1)\n",
      "543 (1, 15, 5, 1)\n",
      "544 (1, 15, 5, 1)\n",
      "545 (1, 15, 5, 1)\n",
      "546 (1, 15, 5, 1)\n",
      "547 (1, 15, 5, 1)\n",
      "548 (1, 15, 5, 1)\n",
      "549 (2, 15, 5, 1)\n",
      "550 (1, 15, 5, 1)\n",
      "551 (4, 15, 5, 1)\n",
      "552 (2, 15, 5, 1)\n",
      "553 (1, 15, 5, 1)\n",
      "554 (2, 15, 5, 1)\n",
      "555 (1, 15, 5, 1)\n",
      "556 (1, 15, 5, 1)\n",
      "557 (1, 15, 5, 1)\n",
      "558 (1, 15, 5, 1)\n",
      "559 (6, 15, 5, 1)\n",
      "560 (6, 15, 5, 1)\n",
      "561 (1, 15, 5, 1)\n",
      "562 (1, 15, 5, 1)\n",
      "563 (1, 15, 5, 1)\n",
      "564 (1, 15, 5, 1)\n",
      "565 (1, 15, 5, 1)\n",
      "566 (1, 15, 5, 1)\n",
      "567 (1, 15, 5, 1)\n",
      "568 (1, 15, 5, 1)\n",
      "569 (1, 15, 5, 1)\n",
      "570 (1, 15, 5, 1)\n",
      "571 (1, 15, 5, 1)\n",
      "572 (6, 15, 5, 1)\n",
      "573 (1, 15, 5, 1)\n",
      "574 (2, 15, 5, 1)\n",
      "575 (3, 15, 5, 1)\n",
      "576 (3, 15, 5, 1)\n",
      "577 (1, 15, 5, 1)\n",
      "578 (1, 15, 5, 1)\n",
      "579 (3, 15, 5, 1)\n",
      "580 (1, 15, 5, 1)\n",
      "581 (10, 15, 5, 1)\n",
      "582 (1, 15, 5, 1)\n",
      "583 (1, 15, 5, 1)\n",
      "584 (1, 15, 5, 1)\n",
      "585 (1, 15, 5, 1)\n",
      "586 (2, 15, 5, 1)\n",
      "587 (2, 15, 5, 1)\n",
      "588 (3, 15, 5, 1)\n",
      "589 (1, 15, 5, 1)\n",
      "590 (1, 15, 5, 1)\n",
      "591 (1, 15, 5, 1)\n",
      "592 (1, 15, 5, 1)\n",
      "593 (1, 15, 5, 1)\n",
      "594 (1, 15, 5, 1)\n",
      "595 (3, 15, 5, 1)\n",
      "596 (1, 15, 5, 1)\n",
      "597 (1, 15, 5, 1)\n",
      "598 (1, 15, 5, 1)\n",
      "599 (1, 15, 5, 1)\n",
      "600 (1, 15, 5, 1)\n",
      "601 (1, 15, 5, 1)\n",
      "602 (7, 15, 5, 1)\n",
      "603 (1, 15, 5, 1)\n",
      "604 (1, 15, 5, 1)\n",
      "605 (2, 15, 5, 1)\n",
      "606 (1, 15, 5, 1)\n",
      "607 (4, 15, 5, 1)\n",
      "608 (1, 15, 5, 1)\n",
      "609 (1, 15, 5, 1)\n",
      "610 (3, 15, 5, 1)\n",
      "611 (1, 15, 5, 1)\n",
      "612 (4, 15, 5, 1)\n",
      "613 (1, 15, 5, 1)\n",
      "614 (6, 15, 5, 1)\n",
      "615 (3, 15, 5, 1)\n",
      "616 (1, 15, 5, 1)\n",
      "617 (8, 15, 5, 1)\n",
      "618 (6, 15, 5, 1)\n",
      "619 (1, 15, 5, 1)\n",
      "620 (1, 15, 5, 1)\n",
      "621 (2, 15, 5, 1)\n",
      "622 (1, 15, 5, 1)\n",
      "623 (3, 15, 5, 1)\n",
      "624 (1, 15, 5, 1)\n",
      "625 (1, 15, 5, 1)\n",
      "626 (1, 15, 5, 1)\n",
      "627 (1, 15, 5, 1)\n",
      "628 (4, 15, 5, 1)\n",
      "629 (1, 15, 5, 1)\n",
      "630 (1, 15, 5, 1)\n",
      "631 (2, 15, 5, 1)\n",
      "632 (1, 15, 5, 1)\n",
      "633 (1, 15, 5, 1)\n",
      "634 (4, 15, 5, 1)\n",
      "635 (1, 15, 5, 1)\n",
      "636 (1, 15, 5, 1)\n",
      "637 (1, 15, 5, 1)\n",
      "638 (1, 15, 5, 1)\n",
      "639 (2, 15, 5, 1)\n",
      "640 (1, 15, 5, 1)\n",
      "641 (1, 15, 5, 1)\n",
      "642 (1, 15, 5, 1)\n",
      "643 (1, 15, 5, 1)\n",
      "644 (1, 15, 5, 1)\n",
      "645 (2, 15, 5, 1)\n",
      "646 (1, 15, 5, 1)\n",
      "647 (1, 15, 5, 1)\n",
      "648 (1, 15, 5, 1)\n",
      "649 (1, 15, 5, 1)\n",
      "650 (1, 15, 5, 1)\n",
      "651 (2, 15, 5, 1)\n",
      "652 (1, 15, 5, 1)\n",
      "653 (1, 15, 5, 1)\n",
      "654 (2, 15, 5, 1)\n",
      "655 (1, 15, 5, 1)\n",
      "656 (2, 15, 5, 1)\n",
      "657 (2, 15, 5, 1)\n",
      "658 (1, 15, 5, 1)\n",
      "659 (1, 15, 5, 1)\n",
      "660 (3, 15, 5, 1)\n",
      "661 (1, 15, 5, 1)\n",
      "662 (1, 15, 5, 1)\n",
      "663 (1, 15, 5, 1)\n",
      "664 (2, 15, 5, 1)\n",
      "665 (2, 15, 5, 1)\n",
      "666 (1, 15, 5, 1)\n",
      "667 (2, 15, 5, 1)\n",
      "668 (1, 15, 5, 1)\n",
      "669 (2, 15, 5, 1)\n",
      "670 (2, 15, 5, 1)\n",
      "671 (1, 15, 5, 1)\n",
      "672 (4, 15, 5, 1)\n",
      "673 (1, 15, 5, 1)\n",
      "674 (1, 15, 5, 1)\n",
      "675 (1, 15, 5, 1)\n",
      "676 (6, 15, 5, 1)\n",
      "677 (1, 15, 5, 1)\n",
      "678 (1, 15, 5, 1)\n",
      "679 (1, 15, 5, 1)\n",
      "680 (1, 15, 5, 1)\n",
      "681 (3, 15, 5, 1)\n",
      "682 (1, 15, 5, 1)\n",
      "683 (3, 15, 5, 1)\n",
      "684 (2, 15, 5, 1)\n",
      "685 (1, 15, 5, 1)\n",
      "686 (1, 15, 5, 1)\n",
      "687 (1, 15, 5, 1)\n",
      "688 (1, 15, 5, 1)\n",
      "689 (1, 15, 5, 1)\n",
      "690 (1, 15, 5, 1)\n",
      "691 (1, 15, 5, 1)\n",
      "692 (1, 15, 5, 1)\n",
      "693 (1, 15, 5, 1)\n",
      "694 (1, 15, 5, 1)\n",
      "695 (1, 15, 5, 1)\n",
      "696 (1, 15, 5, 1)\n",
      "697 (1, 15, 5, 1)\n",
      "698 (1, 15, 5, 1)\n",
      "699 (1, 15, 5, 1)\n",
      "700 (1, 15, 5, 1)\n",
      "701 (1, 15, 5, 1)\n",
      "702 (1, 15, 5, 1)\n",
      "703 (1, 15, 5, 1)\n",
      "704 (1, 15, 5, 1)\n",
      "705 (1, 15, 5, 1)\n",
      "706 (1, 15, 5, 1)\n",
      "707 (1, 15, 5, 1)\n",
      "708 (1, 15, 5, 1)\n",
      "709 (1, 15, 5, 1)\n",
      "710 (1, 15, 5, 1)\n",
      "711 (2, 15, 5, 1)\n",
      "712 (2, 15, 5, 1)\n",
      "713 (1, 15, 5, 1)\n",
      "714 (1, 15, 5, 1)\n",
      "715 (4, 15, 5, 1)\n",
      "716 (1, 15, 5, 1)\n",
      "717 (1, 15, 5, 1)\n",
      "718 (6, 15, 5, 1)\n",
      "719 (1, 15, 5, 1)\n",
      "720 (1, 15, 5, 1)\n",
      "721 (1, 15, 5, 1)\n",
      "722 (1, 15, 5, 1)\n",
      "723 (1, 15, 5, 1)\n",
      "724 (1, 15, 5, 1)\n",
      "725 (3, 15, 5, 1)\n",
      "726 (1, 15, 5, 1)\n",
      "727 (1, 15, 5, 1)\n",
      "728 (1, 15, 5, 1)\n",
      "729 (1, 15, 5, 1)\n",
      "730 (1, 15, 5, 1)\n",
      "731 (1, 15, 5, 1)\n",
      "732 (1, 15, 5, 1)\n",
      "733 (1, 15, 5, 1)\n",
      "734 (1, 15, 5, 1)\n",
      "735 (1, 15, 5, 1)\n",
      "736 (1, 15, 5, 1)\n",
      "737 (1, 15, 5, 1)\n",
      "738 (1, 15, 5, 1)\n",
      "739 (1, 15, 5, 1)\n",
      "740 (1, 15, 5, 1)\n",
      "741 (1, 15, 5, 1)\n",
      "742 (1, 15, 5, 1)\n",
      "743 (2, 15, 5, 1)\n",
      "744 (2, 15, 5, 1)\n",
      "745 (1, 15, 5, 1)\n",
      "746 (1, 15, 5, 1)\n",
      "747 (1, 15, 5, 1)\n",
      "748 (1, 15, 5, 1)\n",
      "749 (1, 15, 5, 1)\n",
      "750 (2, 15, 5, 1)\n",
      "751 (2, 15, 5, 1)\n",
      "752 (1, 15, 5, 1)\n",
      "753 (1, 15, 5, 1)\n",
      "754 (1, 15, 5, 1)\n",
      "755 (1, 15, 5, 1)\n",
      "756 (2, 15, 5, 1)\n",
      "757 (2, 15, 5, 1)\n",
      "758 (3, 15, 5, 1)\n",
      "759 (1, 15, 5, 1)\n",
      "760 (1, 15, 5, 1)\n",
      "761 (1, 15, 5, 1)\n",
      "762 (1, 15, 5, 1)\n",
      "763 (1, 15, 5, 1)\n",
      "764 (1, 15, 5, 1)\n",
      "765 (1, 15, 5, 1)\n",
      "766 (1, 15, 5, 1)\n",
      "767 (1, 15, 5, 1)\n",
      "768 (2, 15, 5, 1)\n",
      "769 (1, 15, 5, 1)\n",
      "770 (1, 15, 5, 1)\n",
      "771 (1, 15, 5, 1)\n",
      "772 (1, 15, 5, 1)\n",
      "773 (1, 15, 5, 1)\n",
      "774 (1, 15, 5, 1)\n",
      "775 (2, 15, 5, 1)\n",
      "776 (2, 15, 5, 1)\n",
      "777 (2, 15, 5, 1)\n",
      "778 (1, 15, 5, 1)\n",
      "779 (1, 15, 5, 1)\n",
      "780 (1, 15, 5, 1)\n",
      "781 (1, 15, 5, 1)\n",
      "782 (1, 15, 5, 1)\n",
      "783 (1, 15, 5, 1)\n",
      "784 (1, 15, 5, 1)\n",
      "785 (1, 15, 5, 1)\n",
      "786 (2, 15, 5, 1)\n",
      "787 (1, 15, 5, 1)\n",
      "788 (1, 15, 5, 1)\n",
      "789 (2, 15, 5, 1)\n",
      "790 (3, 15, 5, 1)\n",
      "791 (3, 15, 5, 1)\n",
      "792 (3, 15, 5, 1)\n",
      "793 (3, 15, 5, 1)\n",
      "794 (3, 15, 5, 1)\n",
      "795 (3, 15, 5, 1)\n",
      "796 (3, 15, 5, 1)\n",
      "797 (3, 15, 5, 1)\n",
      "798 (3, 15, 5, 1)\n",
      "799 (3, 15, 5, 1)\n",
      "800 (3, 15, 5, 1)\n",
      "801 (3, 15, 5, 1)\n",
      "802 (2, 15, 5, 1)\n",
      "803 (2, 15, 5, 1)\n",
      "804 (2, 15, 5, 1)\n",
      "805 (2, 15, 5, 1)\n",
      "806 (2, 15, 5, 1)\n",
      "807 (1, 15, 5, 1)\n",
      "808 (1, 15, 5, 1)\n",
      "809 (3, 15, 5, 1)\n",
      "810 (1, 15, 5, 1)\n",
      "811 (1, 15, 5, 1)\n",
      "812 (3, 15, 5, 1)\n",
      "813 (1, 15, 5, 1)\n",
      "814 (1, 15, 5, 1)\n",
      "815 (1, 15, 5, 1)\n",
      "816 (3, 15, 5, 1)\n",
      "817 (2, 15, 5, 1)\n",
      "818 (3, 15, 5, 1)\n",
      "819 (1, 15, 5, 1)\n",
      "820 (1, 15, 5, 1)\n",
      "821 (3, 15, 5, 1)\n",
      "822 (3, 15, 5, 1)\n",
      "823 (3, 15, 5, 1)\n",
      "824 (1, 15, 5, 1)\n",
      "825 (1, 15, 5, 1)\n",
      "826 (1, 15, 5, 1)\n",
      "827 (1, 15, 5, 1)\n",
      "828 (1, 15, 5, 1)\n",
      "829 (1, 15, 5, 1)\n",
      "830 (1, 15, 5, 1)\n",
      "831 (1, 15, 5, 1)\n",
      "832 (2, 15, 5, 1)\n",
      "833 (1, 15, 5, 1)\n",
      "834 (1, 15, 5, 1)\n",
      "835 (2, 15, 5, 1)\n",
      "836 (1, 15, 5, 1)\n",
      "837 (1, 15, 5, 1)\n",
      "838 (3, 15, 5, 1)\n",
      "839 (8, 15, 5, 1)\n",
      "840 (1, 15, 5, 1)\n",
      "841 (17, 15, 5, 1)\n",
      "842 (2, 15, 5, 1)\n",
      "843 (2, 15, 5, 1)\n",
      "844 (6, 15, 5, 1)\n",
      "845 (2, 15, 5, 1)\n",
      "846 (6, 15, 5, 1)\n",
      "847 (3, 15, 5, 1)\n",
      "848 (2, 15, 5, 1)\n",
      "849 (4, 15, 5, 1)\n",
      "850 (2, 15, 5, 1)\n",
      "851 (4, 15, 5, 1)\n",
      "852 (1, 15, 5, 1)\n",
      "853 (1, 15, 5, 1)\n",
      "854 (6, 15, 5, 1)\n",
      "855 (1, 15, 5, 1)\n",
      "856 (3, 15, 5, 1)\n",
      "857 (1, 15, 5, 1)\n",
      "858 (1, 15, 5, 1)\n",
      "859 (1, 15, 5, 1)\n",
      "860 (1, 15, 5, 1)\n",
      "861 (4, 15, 5, 1)\n",
      "862 (1, 15, 5, 1)\n",
      "863 (1, 15, 5, 1)\n",
      "864 (1, 15, 5, 1)\n",
      "865 (2, 15, 5, 1)\n",
      "866 (1, 15, 5, 1)\n",
      "867 (2, 15, 5, 1)\n",
      "868 (1, 15, 5, 1)\n",
      "869 (1, 15, 5, 1)\n",
      "870 (1, 15, 5, 1)\n",
      "871 (1, 15, 5, 1)\n",
      "872 (1, 15, 5, 1)\n",
      "873 (3, 15, 5, 1)\n",
      "874 (1, 15, 5, 1)\n",
      "875 (1, 15, 5, 1)\n",
      "876 (2, 15, 5, 1)\n",
      "877 (1, 15, 5, 1)\n",
      "878 (3, 15, 5, 1)\n",
      "879 (1, 15, 5, 1)\n",
      "880 (3, 15, 5, 1)\n",
      "881 (2, 15, 5, 1)\n",
      "882 (1, 15, 5, 1)\n",
      "883 (2, 15, 5, 1)\n",
      "884 (1, 15, 5, 1)\n",
      "885 (1, 15, 5, 1)\n",
      "886 (1, 15, 5, 1)\n",
      "887 (2, 15, 5, 1)\n",
      "888 (1, 15, 5, 1)\n",
      "889 (1, 15, 5, 1)\n",
      "890 (1, 15, 5, 1)\n",
      "891 (1, 15, 5, 1)\n",
      "892 (1, 15, 5, 1)\n",
      "893 (1, 15, 5, 1)\n",
      "894 (2, 15, 5, 1)\n",
      "895 (1, 15, 5, 1)\n",
      "896 (2, 15, 5, 1)\n",
      "897 (2, 15, 5, 1)\n",
      "898 (1, 15, 5, 1)\n",
      "899 (1, 15, 5, 1)\n",
      "900 (1, 15, 5, 1)\n",
      "901 (1, 15, 5, 1)\n",
      "902 (3, 15, 5, 1)\n",
      "903 (4, 15, 5, 1)\n",
      "904 (3, 15, 5, 1)\n",
      "905 (1, 15, 5, 1)\n",
      "906 (1, 15, 5, 1)\n",
      "907 (1, 15, 5, 1)\n",
      "908 (1, 15, 5, 1)\n",
      "909 (1, 15, 5, 1)\n",
      "910 (1, 15, 5, 1)\n",
      "911 (1, 15, 5, 1)\n",
      "912 (1, 15, 5, 1)\n",
      "913 (1, 15, 5, 1)\n",
      "914 (1, 15, 5, 1)\n",
      "915 (1, 15, 5, 1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "916 (1, 15, 5, 1)\n",
      "917 (1, 15, 5, 1)\n",
      "918 (1, 15, 5, 1)\n",
      "919 (1, 15, 5, 1)\n",
      "920 (1, 15, 5, 1)\n",
      "921 (1, 15, 5, 1)\n",
      "922 (1, 15, 5, 1)\n",
      "923 (2, 15, 5, 1)\n",
      "924 (1, 15, 5, 1)\n",
      "925 (2, 15, 5, 1)\n",
      "926 (2, 15, 5, 1)\n",
      "927 (2, 15, 5, 1)\n",
      "928 (2, 15, 5, 1)\n",
      "929 (1, 15, 5, 1)\n",
      "930 (2, 15, 5, 1)\n",
      "931 (1, 15, 5, 1)\n",
      "932 (16, 15, 5, 1)\n",
      "933 (1, 15, 5, 1)\n",
      "934 (1, 15, 5, 1)\n",
      "935 (1, 15, 5, 1)\n",
      "936 (4, 15, 5, 1)\n",
      "937 (1, 15, 5, 1)\n",
      "938 (2, 15, 5, 1)\n",
      "939 (20, 15, 5, 1)\n",
      "940 (2, 15, 5, 1)\n",
      "941 (2, 15, 5, 1)\n",
      "942 (2, 15, 5, 1)\n",
      "943 (1, 15, 5, 1)\n",
      "944 (1, 15, 5, 1)\n",
      "945 (1, 15, 5, 1)\n",
      "946 (1, 15, 5, 1)\n",
      "947 (12, 15, 5, 1)\n",
      "948 (2, 15, 5, 1)\n",
      "949 (12, 15, 5, 1)\n",
      "950 (1, 15, 5, 1)\n",
      "951 (1, 15, 5, 1)\n",
      "952 (4, 15, 5, 1)\n",
      "953 (1, 15, 5, 1)\n",
      "954 (1, 15, 5, 1)\n",
      "955 (1, 15, 5, 1)\n",
      "956 (1, 15, 5, 1)\n",
      "957 (1, 15, 5, 1)\n",
      "958 (1, 15, 5, 1)\n",
      "959 (1, 15, 5, 1)\n",
      "960 (4, 15, 5, 1)\n",
      "961 (1, 15, 5, 1)\n",
      "962 (1, 15, 5, 1)\n",
      "963 (2, 15, 5, 1)\n",
      "964 (2, 15, 5, 1)\n",
      "965 (1, 15, 5, 1)\n",
      "966 (8, 15, 5, 1)\n",
      "967 (3, 15, 5, 1)\n",
      "968 (1, 15, 5, 1)\n",
      "969 (10, 15, 5, 1)\n",
      "970 (2, 15, 5, 1)\n",
      "971 (4, 15, 5, 1)\n",
      "972 (1, 15, 5, 1)\n",
      "973 (4, 15, 5, 1)\n",
      "974 (3, 15, 5, 1)\n",
      "975 (2, 15, 5, 1)\n",
      "976 (1, 15, 5, 1)\n",
      "977 (4, 15, 5, 1)\n",
      "978 (1, 15, 5, 1)\n",
      "979 (2, 15, 5, 1)\n",
      "980 (1, 15, 5, 1)\n",
      "981 (3, 15, 5, 1)\n",
      "982 (2, 15, 5, 1)\n",
      "983 (1, 15, 5, 1)\n",
      "984 (1, 15, 5, 1)\n",
      "985 (1, 15, 5, 1)\n",
      "986 (1, 15, 5, 1)\n",
      "987 (1, 15, 5, 1)\n",
      "988 (1, 15, 5, 1)\n",
      "989 (1, 15, 5, 1)\n",
      "990 (2, 15, 5, 1)\n",
      "991 (1, 15, 5, 1)\n",
      "992 (2, 15, 5, 1)\n",
      "993 (1, 15, 5, 1)\n",
      "994 (6, 15, 5, 1)\n",
      "995 (2, 15, 5, 1)\n",
      "996 (1, 15, 5, 1)\n",
      "997 (1, 15, 5, 1)\n",
      "998 (3, 15, 5, 1)\n",
      "999 (6, 15, 5, 1)\n",
      "1000 (1, 15, 5, 1)\n",
      "1001 (1, 15, 5, 1)\n",
      "1002 (2, 15, 5, 1)\n",
      "1003 (66, 15, 5, 1)\n",
      "1004 (1, 15, 5, 1)\n",
      "1005 (1, 15, 5, 1)\n",
      "1006 (3, 15, 5, 1)\n",
      "1007 (1, 15, 5, 1)\n",
      "1008 (4, 15, 5, 1)\n",
      "1009 (2, 15, 5, 1)\n",
      "1010 (1, 15, 5, 1)\n",
      "1011 (4, 15, 5, 1)\n",
      "1012 (3, 15, 5, 1)\n",
      "1013 (6, 15, 5, 1)\n",
      "1014 (1, 15, 5, 1)\n",
      "1015 (1, 15, 5, 1)\n",
      "1016 (1, 15, 5, 1)\n",
      "1017 (6, 15, 5, 1)\n",
      "1018 (1, 15, 5, 1)\n",
      "1019 (10, 15, 5, 1)\n",
      "1020 (1, 15, 5, 1)\n",
      "1021 (2, 15, 5, 1)\n",
      "1022 (1, 15, 5, 1)\n",
      "1023 (1, 15, 5, 1)\n",
      "1024 (1, 15, 5, 1)\n",
      "1025 (2, 15, 5, 1)\n",
      "1026 (1, 15, 5, 1)\n",
      "1027 (2, 15, 5, 1)\n",
      "1028 (1, 15, 5, 1)\n",
      "1029 (1, 15, 5, 1)\n",
      "1030 (1, 15, 5, 1)\n",
      "1031 (1, 15, 5, 1)\n",
      "1032 (1, 15, 5, 1)\n",
      "1033 (1, 15, 5, 1)\n",
      "1034 (1, 15, 5, 1)\n",
      "1035 (1, 15, 5, 1)\n",
      "1036 (1, 15, 5, 1)\n",
      "1037 (1, 15, 5, 1)\n",
      "1038 (1, 15, 5, 1)\n",
      "1039 (2, 15, 5, 1)\n",
      "1040 (1, 15, 5, 1)\n",
      "1041 (3, 15, 5, 1)\n",
      "1042 (4, 15, 5, 1)\n",
      "1043 (2, 15, 5, 1)\n",
      "1044 (1, 15, 5, 1)\n",
      "1045 (1, 15, 5, 1)\n",
      "1046 (1, 15, 5, 1)\n",
      "1047 (1, 15, 5, 1)\n",
      "1048 (1, 15, 5, 1)\n",
      "1049 (1, 15, 5, 1)\n",
      "1050 (1, 15, 5, 1)\n",
      "1051 (1, 15, 5, 1)\n",
      "1052 (1, 15, 5, 1)\n",
      "1053 (1, 15, 5, 1)\n",
      "1054 (1, 15, 5, 1)\n",
      "1055 (1, 15, 5, 1)\n",
      "1056 (1, 15, 5, 1)\n",
      "1057 (1, 15, 5, 1)\n",
      "1058 (1, 15, 5, 1)\n",
      "1059 (1, 15, 5, 1)\n",
      "1060 (1, 15, 5, 1)\n",
      "1061 (1, 15, 5, 1)\n",
      "1062 (8, 15, 5, 1)\n",
      "1063 (1, 15, 5, 1)\n",
      "1064 (1, 15, 5, 1)\n",
      "1065 (1, 15, 5, 1)\n",
      "1066 (1, 15, 5, 1)\n",
      "1067 (1, 15, 5, 1)\n",
      "1068 (6, 15, 5, 1)\n",
      "1069 (1, 15, 5, 1)\n",
      "1070 (1, 15, 5, 1)\n",
      "1071 (2, 15, 5, 1)\n",
      "1072 (2, 15, 5, 1)\n",
      "1073 (4, 15, 5, 1)\n",
      "1074 (2, 15, 5, 1)\n",
      "1075 (1, 15, 5, 1)\n",
      "1076 (3, 15, 5, 1)\n",
      "1077 (1, 15, 5, 1)\n",
      "1078 (1, 15, 5, 1)\n",
      "1079 (1, 15, 5, 1)\n",
      "1080 (1, 15, 5, 1)\n",
      "1081 (15, 15, 5, 1)\n",
      "1082 (2, 15, 5, 1)\n",
      "1083 (4, 15, 5, 1)\n",
      "1084 (4, 15, 5, 1)\n",
      "1085 (4, 15, 5, 1)\n",
      "1086 (15, 15, 5, 1)\n",
      "1087 (1, 15, 5, 1)\n",
      "1088 (2, 15, 5, 1)\n",
      "1089 (2, 15, 5, 1)\n",
      "1090 (1, 15, 5, 1)\n",
      "1091 (2, 15, 5, 1)\n",
      "1092 (5, 15, 5, 1)\n",
      "1093 (1, 15, 5, 1)\n",
      "1094 (2, 15, 5, 1)\n",
      "1095 (3, 15, 5, 1)\n",
      "1096 (1, 15, 5, 1)\n",
      "1097 (1, 15, 5, 1)\n",
      "1098 (1, 15, 5, 1)\n",
      "1099 (1, 15, 5, 1)\n",
      "1100 (2, 15, 5, 1)\n",
      "1101 (1, 15, 5, 1)\n",
      "1102 (1, 15, 5, 1)\n",
      "1103 (1, 15, 5, 1)\n",
      "1104 (1, 15, 5, 1)\n",
      "1105 (1, 15, 5, 1)\n",
      "1106 (1, 15, 5, 1)\n",
      "1107 (1, 15, 5, 1)\n",
      "1108 (1, 15, 5, 1)\n",
      "1109 (1, 15, 5, 1)\n",
      "1110 (1, 15, 5, 1)\n",
      "1111 (1, 15, 5, 1)\n",
      "1112 (2, 15, 5, 1)\n",
      "1113 (1, 15, 5, 1)\n",
      "1114 (13, 15, 5, 1)\n",
      "1115 (2, 15, 5, 1)\n",
      "1116 (8, 15, 5, 1)\n",
      "1117 (2, 15, 5, 1)\n",
      "1118 (1, 15, 5, 1)\n",
      "1119 (1, 15, 5, 1)\n",
      "1120 (1, 15, 5, 1)\n",
      "1121 (1, 15, 5, 1)\n",
      "1122 (1, 15, 5, 1)\n",
      "1123 (2, 15, 5, 1)\n",
      "1124 (1, 15, 5, 1)\n",
      "1125 (2, 15, 5, 1)\n",
      "1126 (1, 15, 5, 1)\n",
      "1127 (1, 15, 5, 1)\n",
      "1128 (1, 15, 5, 1)\n",
      "1129 (1, 15, 5, 1)\n",
      "1130 (1, 15, 5, 1)\n",
      "1131 (1, 15, 5, 1)\n",
      "1132 (1, 15, 5, 1)\n",
      "1133 (1, 15, 5, 1)\n",
      "1134 (1, 15, 5, 1)\n",
      "1135 (1, 15, 5, 1)\n",
      "1136 (1, 15, 5, 1)\n",
      "1137 (1, 15, 5, 1)\n",
      "1138 (1, 15, 5, 1)\n",
      "1139 (1, 15, 5, 1)\n",
      "1140 (2, 15, 5, 1)\n",
      "1141 (2, 15, 5, 1)\n",
      "1142 (1, 15, 5, 1)\n",
      "1143 (1, 15, 5, 1)\n",
      "1144 (1, 15, 5, 1)\n",
      "1145 (1, 15, 5, 1)\n",
      "1146 (1, 15, 5, 1)\n",
      "1147 (1, 15, 5, 1)\n",
      "1148 (1, 15, 5, 1)\n",
      "1149 (1, 15, 5, 1)\n",
      "1150 (1, 15, 5, 1)\n",
      "1151 (1, 15, 5, 1)\n",
      "1152 (2, 15, 5, 1)\n",
      "1153 (1, 15, 5, 1)\n",
      "1154 (1, 15, 5, 1)\n",
      "1155 (1, 15, 5, 1)\n",
      "1156 (1, 15, 5, 1)\n",
      "1157 (4, 15, 5, 1)\n",
      "1158 (4, 15, 5, 1)\n",
      "1159 (4, 15, 5, 1)\n",
      "1160 (1, 15, 5, 1)\n",
      "1161 (1, 15, 5, 1)\n",
      "1162 (2, 15, 5, 1)\n",
      "1163 (1, 15, 5, 1)\n",
      "1164 (1, 15, 5, 1)\n",
      "1165 (1, 15, 5, 1)\n",
      "1166 (2, 15, 5, 1)\n",
      "1167 (1, 15, 5, 1)\n",
      "1168 (1, 15, 5, 1)\n",
      "1169 (1, 15, 5, 1)\n",
      "1170 (1, 15, 5, 1)\n",
      "1171 (2, 15, 5, 1)\n",
      "1172 (2, 15, 5, 1)\n",
      "1173 (1, 15, 5, 1)\n",
      "1174 (1, 15, 5, 1)\n",
      "1175 (1, 15, 5, 1)\n",
      "1176 (1, 15, 5, 1)\n",
      "1177 (1, 15, 5, 1)\n",
      "1178 (1, 15, 5, 1)\n",
      "1179 (1, 15, 5, 1)\n",
      "1180 (1, 15, 5, 1)\n",
      "1181 (1, 15, 5, 1)\n",
      "1182 (1, 15, 5, 1)\n",
      "1183 (1, 15, 5, 1)\n",
      "1184 (1, 15, 5, 1)\n",
      "1185 (5, 15, 5, 1)\n",
      "1186 (1, 15, 5, 1)\n",
      "1187 (1, 15, 5, 1)\n",
      "1188 (1, 15, 5, 1)\n",
      "1189 (1, 15, 5, 1)\n",
      "1190 (1, 15, 5, 1)\n",
      "1191 (1, 15, 5, 1)\n",
      "1192 (2, 15, 5, 1)\n",
      "1193 (2, 15, 5, 1)\n",
      "1194 (1, 15, 5, 1)\n",
      "1195 (2, 15, 5, 1)\n",
      "1196 (2, 15, 5, 1)\n",
      "1197 (1, 15, 5, 1)\n",
      "1198 (1, 15, 5, 1)\n",
      "1199 (2, 15, 5, 1)\n",
      "1200 (1, 15, 5, 1)\n",
      "1201 (1, 15, 5, 1)\n",
      "1202 (2, 15, 5, 1)\n",
      "1203 (1, 15, 5, 1)\n",
      "1204 (1, 15, 5, 1)\n",
      "1205 (1, 15, 5, 1)\n",
      "1206 (1, 15, 5, 1)\n",
      "1207 (1, 15, 5, 1)\n",
      "1208 (1, 15, 5, 1)\n",
      "1209 (1, 15, 5, 1)\n",
      "1210 (1, 15, 5, 1)\n",
      "1211 (1, 15, 5, 1)\n",
      "1212 (1, 15, 5, 1)\n",
      "1213 (1, 15, 5, 1)\n",
      "1214 (1, 15, 5, 1)\n",
      "1215 (1, 15, 5, 1)\n",
      "1216 (1, 15, 5, 1)\n",
      "1217 (1, 15, 5, 1)\n",
      "1218 (1, 15, 5, 1)\n",
      "1219 (1, 15, 5, 1)\n",
      "1220 (1, 15, 5, 1)\n",
      "1221 (1, 15, 5, 1)\n",
      "1222 (1, 15, 5, 1)\n",
      "1223 (1, 15, 5, 1)\n",
      "1224 (1, 15, 5, 1)\n",
      "1225 (1, 15, 5, 1)\n",
      "1226 (1, 15, 5, 1)\n",
      "1227 (2, 15, 5, 1)\n",
      "1228 (1, 15, 5, 1)\n",
      "1229 (2, 15, 5, 1)\n",
      "1230 (1, 15, 5, 1)\n",
      "1231 (1, 15, 5, 1)\n",
      "1232 (1, 15, 5, 1)\n",
      "1233 (1, 15, 5, 1)\n",
      "1234 (1, 15, 5, 1)\n",
      "1235 (1, 15, 5, 1)\n",
      "1236 (2, 15, 5, 1)\n",
      "1237 (2, 15, 5, 1)\n",
      "1238 (2, 15, 5, 1)\n",
      "1239 (1, 15, 5, 1)\n",
      "1240 (4, 15, 5, 1)\n",
      "1241 (4, 15, 5, 1)\n",
      "1242 (4, 15, 5, 1)\n",
      "1243 (4, 15, 5, 1)\n",
      "1244 (4, 15, 5, 1)\n",
      "1245 (4, 15, 5, 1)\n",
      "1246 (4, 15, 5, 1)\n",
      "1247 (4, 15, 5, 1)\n",
      "1248 (4, 15, 5, 1)\n",
      "1249 (4, 15, 5, 1)\n",
      "1250 (4, 15, 5, 1)\n",
      "1251 (4, 15, 5, 1)\n",
      "1252 (4, 15, 5, 1)\n",
      "1253 (4, 15, 5, 1)\n",
      "1254 (4, 15, 5, 1)\n",
      "1255 (1, 15, 5, 1)\n",
      "1256 (1, 15, 5, 1)\n",
      "1257 (1, 15, 5, 1)\n",
      "1258 (7, 15, 5, 1)\n",
      "1259 (1, 15, 5, 1)\n",
      "1260 (1, 15, 5, 1)\n",
      "1261 (1, 15, 5, 1)\n",
      "1262 (1, 15, 5, 1)\n",
      "1263 (1, 15, 5, 1)\n",
      "1264 (1, 15, 5, 1)\n",
      "1265 (1, 15, 5, 1)\n",
      "1266 (1, 15, 5, 1)\n",
      "1267 (1, 15, 5, 1)\n",
      "1268 (12, 15, 5, 1)\n",
      "1269 (1, 15, 5, 1)\n",
      "1270 (1, 15, 5, 1)\n",
      "1271 (1, 15, 5, 1)\n",
      "1272 (1, 15, 5, 1)\n",
      "1273 (1, 15, 5, 1)\n",
      "1274 (1, 15, 5, 1)\n",
      "1275 (1, 15, 5, 1)\n",
      "1276 (1, 15, 5, 1)\n",
      "1277 (1, 15, 5, 1)\n",
      "1278 (6, 15, 5, 1)\n",
      "1279 (6, 15, 5, 1)\n",
      "1280 (2, 15, 5, 1)\n",
      "1281 (2, 15, 5, 1)\n",
      "1282 (2, 15, 5, 1)\n",
      "1283 (2, 15, 5, 1)\n",
      "1284 (4, 15, 5, 1)\n",
      "1285 (2, 15, 5, 1)\n",
      "1286 (1, 15, 5, 1)\n",
      "1287 (1, 15, 5, 1)\n",
      "1288 (1, 15, 5, 1)\n",
      "1289 (1, 15, 5, 1)\n",
      "1290 (1, 15, 5, 1)\n",
      "1291 (1, 15, 5, 1)\n",
      "1292 (1, 15, 5, 1)\n",
      "1293 (1, 15, 5, 1)\n",
      "1294 (2, 15, 5, 1)\n",
      "1295 (2, 15, 5, 1)\n",
      "1296 (2, 15, 5, 1)\n",
      "1297 (3, 15, 5, 1)\n",
      "1298 (1, 15, 5, 1)\n",
      "1299 (1, 15, 5, 1)\n",
      "1300 (4, 15, 5, 1)\n",
      "1301 (1, 15, 5, 1)\n",
      "1302 (1, 15, 5, 1)\n",
      "1303 (2, 15, 5, 1)\n",
      "1304 (1, 15, 5, 1)\n",
      "1305 (1, 15, 5, 1)\n",
      "1306 (1, 15, 5, 1)\n",
      "1307 (2, 15, 5, 1)\n",
      "1308 (1, 15, 5, 1)\n",
      "1309 (1, 15, 5, 1)\n",
      "1310 (1, 15, 5, 1)\n",
      "1311 (1, 15, 5, 1)\n",
      "1312 (1, 15, 5, 1)\n",
      "1313 (1, 15, 5, 1)\n",
      "1314 (1, 15, 5, 1)\n",
      "1315 (1, 15, 5, 1)\n",
      "1316 (4, 15, 5, 1)\n",
      "1317 (1, 15, 5, 1)\n",
      "1318 (2, 15, 5, 1)\n",
      "1319 (1, 15, 5, 1)\n",
      "1320 (2, 15, 5, 1)\n",
      "1321 (1, 15, 5, 1)\n",
      "1322 (1, 15, 5, 1)\n",
      "1323 (1, 15, 5, 1)\n",
      "1324 (1, 15, 5, 1)\n",
      "1325 (1, 15, 5, 1)\n",
      "1326 (1, 15, 5, 1)\n",
      "1327 (1, 15, 5, 1)\n",
      "1328 (1, 15, 5, 1)\n",
      "1329 (1, 15, 5, 1)\n",
      "1330 (1, 15, 5, 1)\n",
      "1331 (2, 15, 5, 1)\n",
      "1332 (1, 15, 5, 1)\n",
      "1333 (1, 15, 5, 1)\n",
      "1334 (1, 15, 5, 1)\n",
      "1335 (1, 15, 5, 1)\n",
      "1336 (1, 15, 5, 1)\n",
      "1337 (1, 15, 5, 1)\n",
      "1338 (1, 15, 5, 1)\n",
      "1339 (4, 15, 5, 1)\n",
      "1340 (1, 15, 5, 1)\n",
      "1341 (1, 15, 5, 1)\n",
      "1342 (1, 15, 5, 1)\n",
      "1343 (1, 15, 5, 1)\n",
      "1344 (1, 15, 5, 1)\n",
      "1345 (1, 15, 5, 1)\n",
      "1346 (1, 15, 5, 1)\n",
      "1347 (1, 15, 5, 1)\n",
      "1348 (2, 15, 5, 1)\n",
      "1349 (6, 15, 5, 1)\n",
      "1350 (1, 15, 5, 1)\n",
      "1351 (1, 15, 5, 1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1352 (2, 15, 5, 1)\n",
      "1353 (1, 15, 5, 1)\n",
      "1354 (10, 15, 5, 1)\n",
      "1355 (2, 15, 5, 1)\n",
      "1356 (3, 15, 5, 1)\n",
      "1357 (2, 15, 5, 1)\n",
      "1358 (1, 15, 5, 1)\n",
      "1359 (1, 15, 5, 1)\n",
      "1360 (1, 15, 5, 1)\n",
      "1361 (1, 15, 5, 1)\n",
      "1362 (1, 15, 5, 1)\n",
      "1363 (1, 15, 5, 1)\n",
      "1364 (1, 15, 5, 1)\n",
      "1365 (1, 15, 5, 1)\n",
      "1366 (1, 15, 5, 1)\n",
      "1367 (1, 15, 5, 1)\n",
      "1368 (1, 15, 5, 1)\n",
      "1369 (1, 15, 5, 1)\n",
      "1370 (1, 15, 5, 1)\n",
      "1371 (1, 15, 5, 1)\n",
      "1372 (1, 15, 5, 1)\n",
      "1373 (2, 15, 5, 1)\n",
      "1374 (2, 15, 5, 1)\n",
      "1375 (2, 15, 5, 1)\n",
      "1376 (4, 15, 5, 1)\n",
      "1377 (2, 15, 5, 1)\n",
      "1378 (2, 15, 5, 1)\n",
      "1379 (3, 15, 5, 1)\n",
      "1380 (4, 15, 5, 1)\n",
      "1381 (2, 15, 5, 1)\n",
      "1382 (1, 15, 5, 1)\n",
      "1383 (2, 15, 5, 1)\n",
      "1384 (4, 15, 5, 1)\n",
      "1385 (2, 15, 5, 1)\n",
      "1386 (3, 15, 5, 1)\n",
      "1387 (2, 15, 5, 1)\n",
      "1388 (5, 15, 5, 1)\n",
      "1389 (2, 15, 5, 1)\n",
      "1390 (2, 15, 5, 1)\n",
      "1391 (2, 15, 5, 1)\n",
      "1392 (2, 15, 5, 1)\n",
      "1393 (3, 15, 5, 1)\n",
      "1394 (2, 15, 5, 1)\n",
      "1395 (1, 15, 5, 1)\n",
      "1396 (2, 15, 5, 1)\n",
      "1397 (1, 15, 5, 1)\n",
      "1398 (1, 15, 5, 1)\n",
      "1399 (1, 15, 5, 1)\n",
      "1400 (1, 15, 5, 1)\n",
      "1401 (1, 15, 5, 1)\n",
      "1402 (2, 15, 5, 1)\n",
      "1403 (1, 15, 5, 1)\n",
      "1404 (1, 15, 5, 1)\n",
      "1405 (5, 15, 5, 1)\n",
      "1406 (1, 15, 5, 1)\n",
      "1407 (1, 15, 5, 1)\n",
      "1408 (2, 15, 5, 1)\n",
      "1409 (2, 15, 5, 1)\n",
      "1410 (4, 15, 5, 1)\n",
      "1411 (19, 15, 5, 1)\n",
      "1412 (1, 15, 5, 1)\n",
      "1413 (8, 15, 5, 1)\n",
      "1414 (3, 15, 5, 1)\n",
      "1415 (1, 15, 5, 1)\n",
      "1416 (1, 15, 5, 1)\n",
      "1417 (1, 15, 5, 1)\n",
      "1418 (2, 15, 5, 1)\n",
      "1419 (3, 15, 5, 1)\n",
      "1420 (3, 15, 5, 1)\n",
      "1421 (1, 15, 5, 1)\n",
      "1422 (1, 15, 5, 1)\n",
      "1423 (10, 15, 5, 1)\n",
      "1424 (3, 15, 5, 1)\n",
      "1425 (1, 15, 5, 1)\n",
      "1426 (8, 15, 5, 1)\n",
      "1427 (2, 15, 5, 1)\n",
      "1428 (3, 15, 5, 1)\n",
      "1429 (2, 15, 5, 1)\n",
      "1430 (1, 15, 5, 1)\n",
      "1431 (2, 15, 5, 1)\n",
      "1432 (18, 15, 5, 1)\n",
      "1433 (1, 15, 5, 1)\n",
      "1434 (1, 15, 5, 1)\n",
      "1435 (2, 15, 5, 1)\n",
      "1436 (1, 15, 5, 1)\n",
      "1437 (3, 15, 5, 1)\n",
      "1438 (1, 15, 5, 1)\n",
      "1439 (1, 15, 5, 1)\n",
      "1440 (2, 15, 5, 1)\n",
      "1441 (2, 15, 5, 1)\n",
      "1442 (1, 15, 5, 1)\n",
      "1443 (1, 15, 5, 1)\n",
      "1444 (1, 15, 5, 1)\n",
      "1445 (1, 15, 5, 1)\n",
      "1446 (1, 15, 5, 1)\n",
      "1447 (1, 15, 5, 1)\n",
      "1448 (1, 15, 5, 1)\n",
      "1449 (1, 15, 5, 1)\n",
      "1450 (1, 15, 5, 1)\n",
      "1451 (1, 15, 5, 1)\n",
      "1452 (1, 15, 5, 1)\n",
      "1453 (6, 15, 5, 1)\n",
      "1454 (1, 15, 5, 1)\n",
      "1455 (1, 15, 5, 1)\n",
      "1456 (1, 15, 5, 1)\n",
      "1457 (1, 15, 5, 1)\n",
      "1458 (1, 15, 5, 1)\n",
      "1459 (1, 15, 5, 1)\n",
      "1460 (1, 15, 5, 1)\n",
      "1461 (1, 15, 5, 1)\n",
      "1462 (1, 15, 5, 1)\n",
      "1463 (1, 15, 5, 1)\n",
      "1464 (1, 15, 5, 1)\n",
      "1465 (1, 15, 5, 1)\n",
      "1466 (1, 15, 5, 1)\n",
      "1467 (1, 15, 5, 1)\n",
      "1468 (1, 15, 5, 1)\n",
      "1469 (1, 15, 5, 1)\n",
      "1470 (1, 15, 5, 1)\n",
      "1471 (2, 15, 5, 1)\n",
      "1472 (1, 15, 5, 1)\n",
      "1473 (10, 15, 5, 1)\n",
      "1474 (2, 15, 5, 1)\n",
      "1475 (1, 15, 5, 1)\n",
      "1476 (3, 15, 5, 1)\n",
      "1477 (1, 15, 5, 1)\n",
      "1478 (1, 15, 5, 1)\n",
      "1479 (1, 15, 5, 1)\n",
      "1480 (1, 15, 5, 1)\n",
      "1481 (1, 15, 5, 1)\n",
      "1482 (1, 15, 5, 1)\n",
      "1483 (1, 15, 5, 1)\n",
      "1484 (1, 15, 5, 1)\n",
      "1485 (1, 15, 5, 1)\n",
      "1486 (1, 15, 5, 1)\n",
      "1487 (1, 15, 5, 1)\n",
      "1488 (1, 15, 5, 1)\n",
      "1489 (1, 15, 5, 1)\n",
      "1490 (2, 15, 5, 1)\n",
      "1491 (1, 15, 5, 1)\n",
      "1492 (1, 15, 5, 1)\n",
      "1493 (1, 15, 5, 1)\n",
      "1494 (1, 15, 5, 1)\n",
      "1495 (1, 15, 5, 1)\n",
      "1496 (1, 15, 5, 1)\n",
      "1497 (1, 15, 5, 1)\n",
      "1498 (1, 15, 5, 1)\n",
      "1499 (2, 15, 5, 1)\n",
      "1500 (1, 15, 5, 1)\n",
      "1501 (1, 15, 5, 1)\n",
      "1502 (1, 15, 5, 1)\n",
      "1503 (1, 15, 5, 1)\n",
      "1504 (1, 15, 5, 1)\n",
      "1505 (1, 15, 5, 1)\n",
      "1506 (1, 15, 5, 1)\n",
      "1507 (1, 15, 5, 1)\n",
      "1508 (3, 15, 5, 1)\n",
      "1509 (2, 15, 5, 1)\n",
      "1510 (1, 15, 5, 1)\n",
      "1511 (1, 15, 5, 1)\n",
      "1512 (1, 15, 5, 1)\n",
      "1513 (1, 15, 5, 1)\n",
      "1514 (1, 15, 5, 1)\n",
      "1515 (1, 15, 5, 1)\n",
      "1516 (1, 15, 5, 1)\n",
      "1517 (2, 15, 5, 1)\n",
      "1518 (1, 15, 5, 1)\n",
      "1519 (1, 15, 5, 1)\n",
      "1520 (1, 15, 5, 1)\n",
      "1521 (1, 15, 5, 1)\n",
      "1522 (6, 15, 5, 1)\n",
      "1523 (3, 15, 5, 1)\n",
      "1524 (1, 15, 5, 1)\n",
      "1525 (1, 15, 5, 1)\n",
      "1526 (1, 15, 5, 1)\n",
      "1527 (1, 15, 5, 1)\n",
      "1528 (2, 15, 5, 1)\n",
      "1529 (1, 15, 5, 1)\n",
      "1530 (1, 15, 5, 1)\n",
      "1531 (1, 15, 5, 1)\n",
      "1532 (1, 15, 5, 1)\n",
      "1533 (1, 15, 5, 1)\n",
      "1534 (1, 15, 5, 1)\n",
      "1535 (1, 15, 5, 1)\n",
      "1536 (1, 15, 5, 1)\n",
      "1537 (2, 15, 5, 1)\n",
      "1538 (1, 15, 5, 1)\n",
      "1539 (1, 15, 5, 1)\n",
      "1540 (1, 15, 5, 1)\n",
      "1541 (1, 15, 5, 1)\n",
      "1542 (1, 15, 5, 1)\n",
      "1543 (1, 15, 5, 1)\n",
      "1544 (1, 15, 5, 1)\n",
      "1545 (1, 15, 5, 1)\n",
      "1546 (1, 15, 5, 1)\n",
      "1547 (1, 15, 5, 1)\n",
      "1548 (1, 15, 5, 1)\n",
      "1549 (1, 15, 5, 1)\n",
      "1550 (1, 15, 5, 1)\n",
      "1551 (1, 15, 5, 1)\n",
      "1552 (1, 15, 5, 1)\n",
      "1553 (1, 15, 5, 1)\n",
      "1554 (1, 15, 5, 1)\n",
      "1555 (1, 15, 5, 1)\n",
      "1556 (1, 15, 5, 1)\n",
      "1557 (1, 15, 5, 1)\n",
      "1558 (1, 15, 5, 1)\n",
      "1559 (1, 15, 5, 1)\n",
      "1560 (1, 15, 5, 1)\n",
      "1561 (1, 15, 5, 1)\n",
      "1562 (1, 15, 5, 1)\n",
      "1563 (2, 15, 5, 1)\n",
      "1564 (1, 15, 5, 1)\n",
      "1565 (1, 15, 5, 1)\n",
      "1566 (1, 15, 5, 1)\n",
      "1567 (1, 15, 5, 1)\n",
      "1568 (1, 15, 5, 1)\n",
      "1569 (1, 15, 5, 1)\n",
      "1570 (1, 15, 5, 1)\n",
      "1571 (1, 15, 5, 1)\n",
      "1572 (1, 15, 5, 1)\n",
      "1573 (1, 15, 5, 1)\n",
      "1574 (1, 15, 5, 1)\n",
      "1575 (1, 15, 5, 1)\n",
      "1576 (1, 15, 5, 1)\n",
      "1577 (1, 15, 5, 1)\n",
      "1578 (2, 15, 5, 1)\n",
      "1579 (1, 15, 5, 1)\n",
      "1580 (1, 15, 5, 1)\n",
      "1581 (1, 15, 5, 1)\n",
      "1582 (1, 15, 5, 1)\n",
      "1583 (1, 15, 5, 1)\n",
      "1584 (1, 15, 5, 1)\n",
      "1585 (1, 15, 5, 1)\n",
      "1586 (1, 15, 5, 1)\n",
      "1587 (1, 15, 5, 1)\n",
      "1588 (1, 15, 5, 1)\n",
      "1589 (1, 15, 5, 1)\n",
      "1590 (1, 15, 5, 1)\n",
      "1591 (1, 15, 5, 1)\n",
      "1592 (2, 15, 5, 1)\n",
      "1593 (1, 15, 5, 1)\n",
      "1594 (1, 15, 5, 1)\n",
      "1595 (1, 15, 5, 1)\n",
      "1596 (1, 15, 5, 1)\n",
      "1597 (1, 15, 5, 1)\n",
      "1598 (2, 15, 5, 1)\n",
      "1599 (3, 15, 5, 1)\n",
      "1600 (6, 15, 5, 1)\n",
      "1601 (1, 15, 5, 1)\n",
      "1602 (1, 15, 5, 1)\n",
      "1603 (4, 15, 5, 1)\n",
      "1604 (1, 15, 5, 1)\n",
      "1605 (1, 15, 5, 1)\n",
      "1606 (1, 15, 5, 1)\n",
      "1607 (2, 15, 5, 1)\n",
      "1608 (1, 15, 5, 1)\n",
      "1609 (1, 15, 5, 1)\n",
      "1610 (2, 15, 5, 1)\n",
      "1611 (1, 15, 5, 1)\n",
      "1612 (1, 15, 5, 1)\n",
      "1613 (1, 15, 5, 1)\n",
      "1614 (1, 15, 5, 1)\n",
      "1615 (1, 15, 5, 1)\n",
      "1616 (1, 15, 5, 1)\n",
      "1617 (1, 15, 5, 1)\n",
      "1618 (1, 15, 5, 1)\n",
      "1619 (1, 15, 5, 1)\n",
      "1620 (1, 15, 5, 1)\n",
      "1621 (1, 15, 5, 1)\n",
      "1622 (1, 15, 5, 1)\n",
      "1623 (4, 15, 5, 1)\n",
      "1624 (1, 15, 5, 1)\n",
      "1625 (2, 15, 5, 1)\n",
      "1626 (2, 15, 5, 1)\n",
      "1627 (2, 15, 5, 1)\n",
      "1628 (1, 15, 5, 1)\n",
      "1629 (12, 15, 5, 1)\n",
      "1630 (2, 15, 5, 1)\n",
      "1631 (2, 15, 5, 1)\n",
      "1632 (2, 15, 5, 1)\n",
      "1633 (1, 15, 5, 1)\n",
      "1634 (9, 15, 5, 1)\n",
      "1635 (1, 15, 5, 1)\n",
      "1636 (1, 15, 5, 1)\n",
      "1637 (1, 15, 5, 1)\n",
      "1638 (2, 15, 5, 1)\n",
      "1639 (1, 15, 5, 1)\n",
      "1640 (1, 15, 5, 1)\n"
     ]
    }
   ],
   "source": [
    "baseline_f = pd.read_csv('/home/vibhav_student/2Dcnn/data/w_all_data/baseline_f_tq-tkn.csv')\n",
    "baseline_f = TREC_data_prep(config, vocab).pipeline(baseline_f.iloc[:,:])\n",
    "baseline_f.reset_index(drop=True,inplace=True) \n",
    "# baseline_f.to_csv('/home/vibhav_student/2Dcnn/data/w_all_data/baseline_f_tq-tkn-pad.csv',index=False)\n",
    "savepkl('/home/vibhav_student/2Dcnn/data/w_all_data/baseline_f_tq-tkn-pad_qfix_shuffle.csv',baseline_f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>']]\n",
      "[['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>']]\n",
      "[['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>']]\n",
      "[['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>']]\n",
      "[['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>']]\n",
      "[['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>']]\n",
      "[['<PAD>'], ['no_credit_rating'], ['the_lender'], ['<PAD>'], ['<PAD>']]\n",
      "[['<PAD>'], ['high_risk'], ['<UNK>'], ['<PAD>'], ['<PAD>']]\n",
      "[['<PAD>'], ['risk'], ['<UNK>'], ['<PAD>'], ['<PAD>']]\n",
      "[['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>']]\n",
      "[['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>']]\n",
      "[['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>']]\n",
      "[['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>']]\n",
      "[['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>']]\n",
      "[['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>'], ['<PAD>']]\n"
     ]
    }
   ],
   "source": [
    "print_tableIDs(baseline_f['table_pad'][0],vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare the model and features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# baseline_f = pd.read_csv('/home/vibhav_student/2Dcnn/data/w_all_data/baseline_f_tq-tkn-pad.csv')\n",
    "# baseline_f['table_pad'] = baseline_f['table_pad'].apply(eval)\n",
    "# baseline_f['query_pad'] = baseline_f['query_pad'].apply(eval)\n",
    "baseline_f = loadpkl('/home/vibhav_student/2Dcnn/data/w_all_data/baseline_f_tq-tkn-pad_qfix_shuffle.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Table2Vec(\n",
       "  (embeddings): Embedding(2533532, 100)\n",
       "  (cnn_layers): Sequential(\n",
       "    (0): Conv2d(100, 128, kernel_size=(3, 2), stride=(1, 1))\n",
       "    (1): ReLU(inplace=True)\n",
       "    (2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "  )\n",
       "  (linear_layers): Sequential(\n",
       "    (0): Linear(in_features=1536, out_features=1, bias=True)\n",
       "    (1): Sigmoid()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = models.create_model(\n",
    "    config['model_props']['type'],\n",
    "    params=(len(vocab), config['model_params']['embedding_dim'])\n",
    ")\n",
    "model.to(device)\n",
    "state_dict = torch.load(os.path.join(args.path, args.model_name))\n",
    "model.linear_layers = torch.nn.Sequential(*(list(model.linear_layers.children())[:2]))\n",
    "state_dict_ = OrderedDict({i: state_dict[i] for i in list(model.state_dict())})\n",
    "model.load_state_dict(state_dict_)\n",
    "print(torch.equal(list(model.parameters())[0], state_dict_['embeddings.weight']))\n",
    "# print(model)\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def cos_sim(table, query):\n",
    "        sim = cosine_similarity(np.array(table).reshape(1, -1), np.array(query).reshape(1, -1))\n",
    "        return np.array(sim).reshape(-1)[0]\n",
    "\n",
    "def get_ft(inp, device):\n",
    "    model_inp = torch.tensor(inp.tolist()).to(device)\n",
    "    return model(model_inp).cpu().detach().numpy().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "baseline_f['table_ft'] = get_ft(baseline_f['table_pad'],device)\n",
    "baseline_f['query_ft'] = get_ft(baseline_f['query_pad'],device)\n",
    "baseline_f['cos_sim'] = baseline_f.apply(lambda x: cos_sim(x['table_ft'], x['query_ft']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5596, 50)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseline_f.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query_id</th>\n",
       "      <th>query</th>\n",
       "      <th>table_id</th>\n",
       "      <th>row</th>\n",
       "      <th>col</th>\n",
       "      <th>nul</th>\n",
       "      <th>in_link</th>\n",
       "      <th>out_link</th>\n",
       "      <th>pgcount</th>\n",
       "      <th>tImp</th>\n",
       "      <th>...</th>\n",
       "      <th>resim</th>\n",
       "      <th>query_l</th>\n",
       "      <th>rel</th>\n",
       "      <th>table_tkn</th>\n",
       "      <th>query_tkn</th>\n",
       "      <th>table_pad</th>\n",
       "      <th>query_pad</th>\n",
       "      <th>table_ft</th>\n",
       "      <th>query_ft</th>\n",
       "      <th>cos_sim</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>world interest rates Table</td>\n",
       "      <td>table-0875-680</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>21</td>\n",
       "      <td>51438</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.281130</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>[[['risk'], []], [[], []], [[], []], [[], []],...</td>\n",
       "      <td>['world_interest_rates', 'table']</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[0.6962807178497314]</td>\n",
       "      <td>[0.5665705800056458]</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>world interest rates Table</td>\n",
       "      <td>table-1020-619</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>324</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.710250</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>[[['headline'], [], []], [['core'], [], []], [...</td>\n",
       "      <td>['world_interest_rates', 'table']</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[0.5987322926521301]</td>\n",
       "      <td>[0.5665705800056458]</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>world interest rates Table</td>\n",
       "      <td>table-0288-531</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>22</td>\n",
       "      <td>26419</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033680</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>[[['stocks'], [], [], [], []], [['bonds'], [],...</td>\n",
       "      <td>['world_interest_rates', 'table']</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[0.710820734500885]</td>\n",
       "      <td>[0.5665705800056458]</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>world interest rates Table</td>\n",
       "      <td>table-0288-530</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "      <td>22</td>\n",
       "      <td>26419</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033680</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>[[['stocks'], [], [], [], []], [['bonds'], [],...</td>\n",
       "      <td>['world_interest_rates', 'table']</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[0.8574050068855286]</td>\n",
       "      <td>[0.5665705800056458]</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>world interest rates Table</td>\n",
       "      <td>table-1000-57</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>2268</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.279899</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>[[['t_bills'], ['return']], [['15-year_dated_s...</td>\n",
       "      <td>['world_interest_rates', 'table']</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[0.3828888237476349]</td>\n",
       "      <td>[0.5665705800056458]</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5591</th>\n",
       "      <td>60</td>\n",
       "      <td>games age</td>\n",
       "      <td>table-0221-887</td>\n",
       "      <td>22</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>87</td>\n",
       "      <td>73</td>\n",
       "      <td>4080</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>...</td>\n",
       "      <td>0.031775</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>[[[], ['shot_&amp;_shell'], ['roger_nord'], ['worl...</td>\n",
       "      <td>['games_age']</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[0.9868162274360657]</td>\n",
       "      <td>[0.45424747467041016]</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5592</th>\n",
       "      <td>60</td>\n",
       "      <td>games age</td>\n",
       "      <td>table-0221-887</td>\n",
       "      <td>22</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>87</td>\n",
       "      <td>73</td>\n",
       "      <td>4080</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>...</td>\n",
       "      <td>0.031775</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>[[[], ['shot_&amp;_shell'], ['roger_nord'], ['worl...</td>\n",
       "      <td>['games_age']</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[0.9868162274360657]</td>\n",
       "      <td>[0.45424747467041016]</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5593</th>\n",
       "      <td>60</td>\n",
       "      <td>games age</td>\n",
       "      <td>table-0609-879</td>\n",
       "      <td>31</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>579</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.013236</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>[[['a_tale_the_desert'], [], ['egenesis']], [[...</td>\n",
       "      <td>['games_age']</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[0.4297628402709961]</td>\n",
       "      <td>[0.45424747467041016]</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5594</th>\n",
       "      <td>60</td>\n",
       "      <td>games age</td>\n",
       "      <td>table-0609-879</td>\n",
       "      <td>31</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>579</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.013236</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>[[['a_tale_the_desert'], [], ['egenesis']], [[...</td>\n",
       "      <td>['games_age']</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[0.4297628402709961]</td>\n",
       "      <td>[0.45424747467041016]</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5595</th>\n",
       "      <td>60</td>\n",
       "      <td>games age</td>\n",
       "      <td>table-0609-879</td>\n",
       "      <td>31</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>579</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.013236</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>[[['a_tale_the_desert'], [], ['egenesis']], [[...</td>\n",
       "      <td>['games_age']</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[[[2384110], [2384110], [2384110], [2384110], ...</td>\n",
       "      <td>[0.4297628402709961]</td>\n",
       "      <td>[0.45424747467041016]</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5596 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      query_id                       query        table_id  row  col  nul  \\\n",
       "0            1  world interest rates Table  table-0875-680    8    2    0   \n",
       "1            1  world interest rates Table  table-1020-619    4    3    0   \n",
       "2            1  world interest rates Table  table-0288-531    3    5    0   \n",
       "3            1  world interest rates Table  table-0288-530    4    5    1   \n",
       "4            1  world interest rates Table   table-1000-57    2    2    0   \n",
       "...        ...                         ...             ...  ...  ...  ...   \n",
       "5591        60                   games age  table-0221-887   22    4    0   \n",
       "5592        60                   games age  table-0221-887   22    4    0   \n",
       "5593        60                   games age  table-0609-879   31    3    0   \n",
       "5594        60                   games age  table-0609-879   31    3    0   \n",
       "5595        60                   games age  table-0609-879   31    3    0   \n",
       "\n",
       "      in_link  out_link  pgcount      tImp  ...     resim  query_l  rel  \\\n",
       "0          31        21    51438  1.000000  ...  0.281130        4    0   \n",
       "1          18         0      324  1.000000  ...  0.710250        4    0   \n",
       "2          23        22    26419  0.500000  ...  0.033680        4    0   \n",
       "3          23        22    26419  0.500000  ...  0.033680        4    0   \n",
       "4          38         1     2268  1.000000  ...  0.279899        4    0   \n",
       "...       ...       ...      ...       ...  ...       ...      ...  ...   \n",
       "5591       87        73     4080  0.045455  ...  0.031775        2    2   \n",
       "5592       87        73     4080  0.045455  ...  0.031775        2    2   \n",
       "5593       25         1      579  1.000000  ...  0.013236        2    2   \n",
       "5594       25         1      579  1.000000  ...  0.013236        2    2   \n",
       "5595       25         1      579  1.000000  ...  0.013236        2    2   \n",
       "\n",
       "                                              table_tkn  \\\n",
       "0     [[['risk'], []], [[], []], [[], []], [[], []],...   \n",
       "1     [[['headline'], [], []], [['core'], [], []], [...   \n",
       "2     [[['stocks'], [], [], [], []], [['bonds'], [],...   \n",
       "3     [[['stocks'], [], [], [], []], [['bonds'], [],...   \n",
       "4     [[['t_bills'], ['return']], [['15-year_dated_s...   \n",
       "...                                                 ...   \n",
       "5591  [[[], ['shot_&_shell'], ['roger_nord'], ['worl...   \n",
       "5592  [[[], ['shot_&_shell'], ['roger_nord'], ['worl...   \n",
       "5593  [[['a_tale_the_desert'], [], ['egenesis']], [[...   \n",
       "5594  [[['a_tale_the_desert'], [], ['egenesis']], [[...   \n",
       "5595  [[['a_tale_the_desert'], [], ['egenesis']], [[...   \n",
       "\n",
       "                              query_tkn  \\\n",
       "0     ['world_interest_rates', 'table']   \n",
       "1     ['world_interest_rates', 'table']   \n",
       "2     ['world_interest_rates', 'table']   \n",
       "3     ['world_interest_rates', 'table']   \n",
       "4     ['world_interest_rates', 'table']   \n",
       "...                                 ...   \n",
       "5591                      ['games_age']   \n",
       "5592                      ['games_age']   \n",
       "5593                      ['games_age']   \n",
       "5594                      ['games_age']   \n",
       "5595                      ['games_age']   \n",
       "\n",
       "                                              table_pad  \\\n",
       "0     [[[2384110], [2384110], [2384110], [2384110], ...   \n",
       "1     [[[2384110], [2384110], [2384110], [2384110], ...   \n",
       "2     [[[2384110], [2384110], [2384110], [2384110], ...   \n",
       "3     [[[2384110], [2384110], [2384110], [2384110], ...   \n",
       "4     [[[2384110], [2384110], [2384110], [2384110], ...   \n",
       "...                                                 ...   \n",
       "5591  [[[2384110], [2384110], [2384110], [2384110], ...   \n",
       "5592  [[[2384110], [2384110], [2384110], [2384110], ...   \n",
       "5593  [[[2384110], [2384110], [2384110], [2384110], ...   \n",
       "5594  [[[2384110], [2384110], [2384110], [2384110], ...   \n",
       "5595  [[[2384110], [2384110], [2384110], [2384110], ...   \n",
       "\n",
       "                                              query_pad              table_ft  \\\n",
       "0     [[[2384110], [2384110], [2384110], [2384110], ...  [0.6962807178497314]   \n",
       "1     [[[2384110], [2384110], [2384110], [2384110], ...  [0.5987322926521301]   \n",
       "2     [[[2384110], [2384110], [2384110], [2384110], ...   [0.710820734500885]   \n",
       "3     [[[2384110], [2384110], [2384110], [2384110], ...  [0.8574050068855286]   \n",
       "4     [[[2384110], [2384110], [2384110], [2384110], ...  [0.3828888237476349]   \n",
       "...                                                 ...                   ...   \n",
       "5591  [[[2384110], [2384110], [2384110], [2384110], ...  [0.9868162274360657]   \n",
       "5592  [[[2384110], [2384110], [2384110], [2384110], ...  [0.9868162274360657]   \n",
       "5593  [[[2384110], [2384110], [2384110], [2384110], ...  [0.4297628402709961]   \n",
       "5594  [[[2384110], [2384110], [2384110], [2384110], ...  [0.4297628402709961]   \n",
       "5595  [[[2384110], [2384110], [2384110], [2384110], ...  [0.4297628402709961]   \n",
       "\n",
       "                   query_ft  cos_sim  \n",
       "0      [0.5665705800056458]      1.0  \n",
       "1      [0.5665705800056458]      1.0  \n",
       "2      [0.5665705800056458]      1.0  \n",
       "3      [0.5665705800056458]      1.0  \n",
       "4      [0.5665705800056458]      1.0  \n",
       "...                     ...      ...  \n",
       "5591  [0.45424747467041016]      1.0  \n",
       "5592  [0.45424747467041016]      1.0  \n",
       "5593  [0.45424747467041016]      1.0  \n",
       "5594  [0.45424747467041016]      1.0  \n",
       "5595  [0.45424747467041016]      1.0  \n",
       "\n",
       "[5596 rows x 50 columns]"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseline_f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "savepkl('/home/vibhav_student/2Dcnn/data/w_all_data/baseline_f_tq-tkn-pad_qfix_shuffle_cossim.csv',baseline_f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run the trec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TREC_model():\n",
    "    def __init__(self, data, output_dir, config):\n",
    "        self.data = data\n",
    "        self.config = config\n",
    "        self.file_path = os.path.join(output_dir, config['trec']['file_name'])\n",
    "        self.prep_data()\n",
    "        make_dirs(output_dir)\n",
    "\n",
    "    def prep_data(self):\n",
    "        x_bf = ['row', 'col', 'nul', 'in_link', 'out_link', 'pgcount', 'tImp', 'tPF', 'leftColhits', 'SecColhits', 'bodyhits', 'PMI', 'qInPgTitle', 'qInTableTitle', 'yRank', 'csr_score', 'idf1',\n",
    "                'idf2', 'idf3', 'idf4', 'idf5', 'idf6', 'max', 'sum', 'avg', 'sim', 'emax', 'esum', 'eavg', 'esim', 'cmax', 'csum', 'cavg', 'csim', 'remax', 'resum', 'reavg', 'resim', 'query_l']\n",
    "        x_smf = ['cos_sim']\n",
    "        x_f = x_bf\n",
    "        y_f = ['rel']\n",
    "        if self.config['trec']['semantic_f']:\n",
    "            x_f += x_smf\n",
    "\n",
    "        self.X = self.data[x_f]\n",
    "        self.y = self.data[y_f]\n",
    "\n",
    "    def train(self):\n",
    "        kfold = KFold(5, True, 42)\n",
    "        for i, indices in enumerate(kfold.split(self.X)):\n",
    "            train_idx, test_idx = indices\n",
    "            X_train, X_test, y_train, y_test = self.X.iloc[train_idx], self.X.iloc[\n",
    "                test_idx], self.y.iloc[train_idx], self.y.iloc[test_idx]\n",
    "            df = self.makeModel_getdf(X_train, X_test, y_train, y_test)\n",
    "            df.to_csv(f\"{self.file_path}{i}.txt\",\n",
    "                      sep=' ', index=False, header=False)\n",
    "\n",
    "    def makeModel_getdf(self, X_train, X_test, y_train, y_test):\n",
    "        self.clf = RandomForestClassifier(\n",
    "            n_estimators=1000,\n",
    "            max_features=3,\n",
    "            random_state=42)\n",
    "        self.clf.fit(X_train, y_train.values.ravel())\n",
    "        X_test = mp(X_test, self.score_mp, 20)\n",
    "#         print(X_test)\n",
    "        df = self.generate_trec_df(self.generate_filtered_df(X_test, y_test))\n",
    "        return df\n",
    "\n",
    "    def score_mp(self, X_test):\n",
    "        X_test['model_score'] = X_test.apply(\n",
    "            lambda x: self.getScore(x), axis=1)\n",
    "        return X_test\n",
    "\n",
    "    def getScore(self, row):\n",
    "        arr = self.clf.predict_proba(np.array(row).reshape(1, -1))\n",
    "        return arr[0][1] + 2 * arr[0][2]\n",
    "\n",
    "    def generate_filtered_df(self, X, y):\n",
    "        df = pd.concat([\n",
    "            self.data.iloc[list(X.index)][['query_id', 'query', 'table_id']],\n",
    "            X['model_score']\n",
    "        ], axis=1)\n",
    "        return df\n",
    "\n",
    "    def generate_trec_df(self, df):\n",
    "        l = []\n",
    "        dic = dict(df.query_id.value_counts())\n",
    "        for i in dic:\n",
    "            for j in range(1, dic[i] + 1):\n",
    "                l.append(j)\n",
    "\n",
    "        df_temp = pd.DataFrame()\n",
    "        df_temp['query_id'] = df['query_id']\n",
    "        df_temp['Q0'] = 'Q0'\n",
    "        df_temp['table_id'] = df['table_id']\n",
    "        df_temp['rank'] = l\n",
    "        df_temp['score'] = df['model_score']\n",
    "        df_temp['smarttable'] = 'smarttable'\n",
    "        return df_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "trec_path = os.path.join(args.path, f'TREC_results_lstlyr-ft_{args.model_name[:-3]}_shuffle')\n",
    "trec_model = TREC_model(data=baseline_f, output_dir=trec_path, config=config)\n",
    "trec_model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ndcg_pipeline(path, trec_path, query_file_path):\n",
    "    d = pd.DataFrame()\n",
    "    for i in range(5):\n",
    "        d_ = pd.read_csv(f\"{path}{i}.txt\", sep=' ', header=None)\n",
    "        d_.drop(columns=[3], inplace=True)\n",
    "        d = pd.concat([d, d_])\n",
    "\n",
    "    d_sorted = d.sort_values(by=[0, 4], ascending=[True, False])\n",
    "\n",
    "    d_sorted_filtered = pd.DataFrame()\n",
    "    for i in range(1, 61):\n",
    "        d_temp = d_sorted[d_sorted[0] == i]\n",
    "        d_temp = d_temp.drop_duplicates(subset=2, keep=\"last\")\n",
    "        d_temp = d_temp.iloc[:20]\n",
    "        d_sorted_filtered = pd.concat([d_sorted_filtered, d_temp])\n",
    "\n",
    "    l = []\n",
    "    for i in range(60):\n",
    "        for j in range(1, 21):\n",
    "            l.append(j)\n",
    "    d_sorted_filtered[3] = l\n",
    "    d_sorted_filtered = d_sorted_filtered[list(range(6))]\n",
    "    d_sorted_filtered.to_csv(\n",
    "        f'{path}all.txt', sep=' ', index=False, header=False)\n",
    "\n",
    "    command = os.popen(\n",
    "        f\"{trec_path} -m ndcg_cut.5,10,15,20 {query_file_path} {path}all.txt\")\n",
    "    result = command.read()\n",
    "    command.close()\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "ndcg_score = ndcg_pipeline(\n",
    "    trec_model.file_path,\n",
    "    '../trec_eval/trec_eval',\n",
    "    '../global_data/qrels.txt')\n",
    "print(ndcg_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "comment = \"Running 6_12_20_41_11 with lr=3e-4, early stopping and lr scheduler(0.5 reduces), 1CNN+1FC\"\n",
      "ndcg_cut_5            \tall\t0.7698\n",
      "ndcg_cut_10           \tall\t0.7646\n",
      "ndcg_cut_15           \tall\t0.7768\n",
      "ndcg_cut_20           \tall\t0.7987\n"
     ]
    }
   ],
   "source": [
    "!cat ./output/6_23_10_10_22/config.toml | grep comment\n",
    "!../trec_eval/trec_eval -m ndcg_cut.5,10,15,20 ../global_data/qrels.txt ./output/6_23_10_10_22/TREC_results_lstlyr-ft_model_5/LTR_k5_all.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ndcg_mlp import NDCG_MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TREC_model():\n",
    "    def __init__(self, data, output_dir, config):\n",
    "        self.data = data\n",
    "        self.config = config\n",
    "        self.file_path = os.path.join(output_dir, config['trec']['file_name'])\n",
    "        self.prep_data()\n",
    "        make_dirs(output_dir)\n",
    "        self.model = NDCG_MLP.load_from_checkpoint('/home/vibhav_student/2Dcnn/lightning_logs/version_18/checkpoints/epoch=9.ckpt')\n",
    "        self.model.eval()\n",
    "        self.get_scores(self.X)\n",
    "\n",
    "    def prep_data(self):\n",
    "        x_bf = ['row', 'col', 'nul', 'in_link', 'out_link', 'pgcount', 'tImp', 'tPF', 'leftColhits', 'SecColhits', 'bodyhits', 'PMI', 'qInPgTitle', 'qInTableTitle', 'yRank', 'csr_score', 'idf1',\n",
    "                'idf2', 'idf3', 'idf4', 'idf5', 'idf6', 'max', 'sum', 'avg', 'sim', 'emax', 'esum', 'eavg', 'esim', 'cmax', 'csum', 'cavg', 'csim', 'remax', 'resum', 'reavg', 'resim', 'query_l']\n",
    "        x_smf = ['cos_sim']\n",
    "        x_f = x_bf\n",
    "        y_f = ['rel']\n",
    "        if self.config['trec']['semantic_f']:\n",
    "            x_f += x_smf\n",
    "\n",
    "        self.X = self.data[x_f]\n",
    "        self.y = self.data[y_f]\n",
    "\n",
    "    def get_scores(self,df):\n",
    "        def get_final_score(arr):\n",
    "            return arr[1] + 2 * arr[2]\n",
    "\n",
    "        val = self.model(torch.tensor(df.to_numpy()).float()).detach().numpy()\n",
    "        df['model_score'] = np.apply_along_axis(get_final_score, 1, val)\n",
    "        df = self.generate_trec_df(self.generate_filtered_df(df))\n",
    "        df.to_csv(f\"{self.file_path}.txt\", sep=' ', index=False, header=False)\n",
    "        \n",
    "    def generate_filtered_df(self, X):\n",
    "        df = pd.concat([\n",
    "            self.data.iloc[list(X.index)][['query_id', 'query', 'table_id']],\n",
    "            X['model_score']\n",
    "        ], axis=1)\n",
    "        return df\n",
    "    \n",
    "    def generate_trec_df(self, df):\n",
    "        l = []\n",
    "        dic = dict(df.query_id.value_counts())\n",
    "        for i in dic:\n",
    "            for j in range(1, dic[i] + 1):\n",
    "                l.append(j)\n",
    "\n",
    "        df_temp = pd.DataFrame()\n",
    "        df_temp['query_id'] = df['query_id']\n",
    "        df_temp['Q0'] = 'Q0'\n",
    "        df_temp['table_id'] = df['table_id']\n",
    "        df_temp['rank'] = l\n",
    "        df_temp['score'] = df['model_score']\n",
    "        df_temp['smarttable'] = 'smarttable'\n",
    "        return df_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5596, 3)\n",
      "0      -4.170529\n",
      "1      -5.077423\n",
      "2      -4.541245\n",
      "3      -4.541245\n",
      "4      -4.541245\n",
      "          ...   \n",
      "5591   -4.541245\n",
      "5592   -4.541245\n",
      "5593   -4.711388\n",
      "5594   -4.711388\n",
      "5595   -4.711388\n",
      "Name: model_score, Length: 5596, dtype: float64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vibhav_student/.conda/envs/t2v_pt/lib/python3.7/site-packages/ipykernel_launcher.py:30: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    }
   ],
   "source": [
    "trec_path = os.path.join(args.path, f'TREC_results_lstlyr-ft_{args.model_name[:-3]}_shuffle_mlp')\n",
    "trec_model = TREC_model(data=baseline_f, output_dir=trec_path, config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ndcg_pipeline(path, trec_path, query_file_path):\n",
    "#     d = pd.DataFrame()\n",
    "#     for i in range(5):\n",
    "#         d_ = pd.read_csv(f\"{path}{i}.txt\", sep=' ', header=None)\n",
    "#         d_.drop(columns=[3], inplace=True)\n",
    "#         d = pd.concat([d, d_])\n",
    "    d = pd.read_csv(f\"{path}.txt\", sep=' ', header=None)\n",
    "\n",
    "    d_sorted = d.sort_values(by=[0, 4], ascending=[True, False])\n",
    "\n",
    "    d_sorted_filtered = pd.DataFrame()\n",
    "    for i in range(1, 61):\n",
    "        d_temp = d_sorted[d_sorted[0] == i]\n",
    "        d_temp = d_temp.drop_duplicates(subset=2, keep=\"last\")\n",
    "        d_temp = d_temp.iloc[:20]\n",
    "        d_sorted_filtered = pd.concat([d_sorted_filtered, d_temp])\n",
    "\n",
    "    l = []\n",
    "    for i in range(60):\n",
    "        for j in range(1, 21):\n",
    "            l.append(j)\n",
    "    d_sorted_filtered[3] = l\n",
    "    d_sorted_filtered = d_sorted_filtered[list(range(6))]\n",
    "    d_sorted_filtered.to_csv(\n",
    "        f'{path}all.txt', sep=' ', index=False, header=False)\n",
    "\n",
    "    command = os.popen(\n",
    "        f\"{trec_path} -m ndcg_cut.5,10,15,20 {query_file_path} {path}all.txt\")\n",
    "    result = command.read()\n",
    "    command.close()\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ndcg_cut_5            \tall\t0.2826\n",
      "ndcg_cut_10           \tall\t0.3393\n",
      "ndcg_cut_15           \tall\t0.3842\n",
      "ndcg_cut_20           \tall\t0.4374\n",
      "\n"
     ]
    }
   ],
   "source": [
    "ndcg_score = ndcg_pipeline(\n",
    "    trec_model.file_path,\n",
    "    '../trec_eval/trec_eval',\n",
    "    '../global_data/qrels.txt')\n",
    "print(ndcg_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ndcg_cut_5            \tall\t0.2826\r\n",
      "ndcg_cut_10           \tall\t0.3393\r\n",
      "ndcg_cut_15           \tall\t0.3842\r\n",
      "ndcg_cut_20           \tall\t0.4374\r\n"
     ]
    }
   ],
   "source": [
    "!../trec_eval/trec_eval -m ndcg_cut.5,10,15,20 ../global_data/qrels.txt /home/vibhav_student/2Dcnn/output/6_29_13_36_22/TREC_results_lstlyr-ft_model_40_shuffle_mlp/LTR_k5_all.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.5 64-bit",
   "language": "python",
   "name": "python37564bitb64b7bd534634d14b355f6a5722a54fe"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}