gen_step_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_VISIBLE_DEVICES=2,3\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%env CUDA_VISIBLE_DEVICES=2,3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import os\n",
    "import random\n",
    "import re\n",
    "import numpy as np\n",
    "import torch\n",
    "\n",
    "def set_seed(seed):\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    \n",
    "    random.seed(seed)\n",
    "    \n",
    "    np.random.seed(seed)\n",
    "    \n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed) # if you are using multi-GPU.\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    \n",
    "def set_global_logging_level(level=logging.ERROR, prefixes=(\"\",)):\n",
    "    \"\"\"\n",
    "    Override logging levels of different modules based on their name as a prefix.\n",
    "    It needs to be invoked after the modules have been loaded so that their loggers have been initialized.\n",
    "\n",
    "    Args:\n",
    "        level: desired level. Optional. Default is logging.ERROR\n",
    "        prefixes: list of one or more str prefixes to match (e.g. [\"transformers\", \"torch\"]). Optional.\n",
    "            Default is `[\"\"]` to match all active loggers.\n",
    "            The match is a case-sensitive `module_name.startswith(prefix)`\n",
    "    \"\"\"\n",
    "    prefix_re = re.compile(fr'^(?:{\"|\".join(prefixes)})')\n",
    "    for name in logging.root.manager.loggerDict:\n",
    "        if re.match(prefix_re, name):\n",
    "            logging.getLogger(name).setLevel(level)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger = logging.getLogger()\n",
    "logger.setLevel(logging.INFO)\n",
    "if logger.hasHandlers():\n",
    "    logger.handlers.clear()\n",
    "console = logging.StreamHandler()\n",
    "logger.addHandler(console)\n",
    "\n",
    "set_seed(0)\n",
    "# set_global_logging_level(logging.WARNING, [\"elasticsearch\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "true"
   },
   "source": [
    "## Load examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "90447\n",
      "90447\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "split = 'train'\n",
    "samples = []\n",
    "with open(f\"data/hotpot-{split}.tsv\") as f:\n",
    "    for line in f:\n",
    "        q_id, question, answer, sp_facts = line.strip().split('\\t')\n",
    "        sp_facts = json.loads(sp_facts)\n",
    "        samples.append((q_id, (question, answer, sp_facts)))\n",
    "print(len(samples))\n",
    "questions = [sample[1][0] for sample in samples]  # (N,)\n",
    "questions = [q[:-1] if q.endswith('?') else q for q in questions]\n",
    "print(len(questions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "true"
   },
   "source": [
    "## Load sparse retriever and query generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch version 1.2.0 available.\n",
      "Loading faiss with AVX2 support.\n",
      "Loading faiss.\n"
     ]
    }
   ],
   "source": [
    "from retriever import SparseRetriever\n",
    "\n",
    "sparse_retriever = SparseRetriever('enwiki-20171001-paragraph-5', ['10.60.0.59:9200'], timeout=60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Initializing model...\n",
      "Loading model ckpts/golden-retriever/hop1.mdl\n",
      "Expanding dictionary...\n",
      "Adding 2115762 new words to dictionary...\n",
      "New vocab size: 2195963\n",
      "Loading pre-trained embeddings for 2115762 words from data/glove.840B.300d.txt\n",
      "WARN: Duplicate embedding found for ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������\n",
      "WARN: Duplicate embedding found for Kṛṣṇa\n",
      "WARN: Duplicate embedding found for Decisión\n",
      "WARN: Duplicate embedding found for decisión\n",
      "WARN: Duplicate embedding found for Viṣṇu\n",
      "WARN: Duplicate embedding found for Justiça\n",
      "WARN: Duplicate embedding found for Acórdão\n",
      "WARN: Duplicate embedding found for ̲̅̅\n",
      "WARN: Duplicate embedding found for Câmara\n",
      "WARN: Duplicate embedding found for brāhmaṇa\n",
      "WARN: Duplicate embedding found for Extensión\n",
      "WARN: Duplicate embedding found for Cível\n",
      "WARN: Duplicate embedding found for brāhmaṇas\n",
      "WARN: Duplicate embedding found for Vṛndāvana\n",
      "WARN: Duplicate embedding found for Cañon\n",
      "WARN: Duplicate embedding found for Arrêté\n",
      "WARN: Duplicate embedding found for kΩ\n",
      "WARN: Duplicate embedding found for Córdova\n",
      "WARN: Duplicate embedding found for justiça\n",
      "WARN: Duplicate embedding found for acórdão\n",
      "WARN: Duplicate embedding found for Ṭhākura\n",
      "WARN: Duplicate embedding found for États\n",
      "WARN: Duplicate embedding found for Nārāyaṇa\n",
      "WARN: Duplicate embedding found for Resolución\n",
      "WARN: Duplicate embedding found for extensión\n",
      "WARN: Duplicate embedding found for ʹ\n",
      "WARN: Duplicate embedding found for Vaiṣṇava\n",
      "WARN: Duplicate embedding found for của\n",
      "WARN: Duplicate embedding found for câmara\n",
      "WARN: Duplicate embedding found for Região\n",
      "WARN: Duplicate embedding found for siècle\n",
      "WARN: Duplicate embedding found for Parīkṣit\n",
      "WARN: Duplicate embedding found for Décima\n",
      "WARN: Duplicate embedding found for mΩ\n",
      "WARN: Duplicate embedding found for Protección\n",
      "WARN: Duplicate embedding found for cível\n",
      "WARN: Duplicate embedding found for Miró\n",
      "WARN: Duplicate embedding found for MΩ\n",
      "WARN: Duplicate embedding found for arrêté\n",
      "WARN: Duplicate embedding found for Tháng\n",
      "Loaded 2115762 embeddings (100.00%)\n",
      "Initializing tokenizer...\n",
      "Initializing model...\n",
      "Loading model ckpts/golden-retriever/hop2.mdl\n",
      "Expanding dictionary...\n",
      "Adding 1892506 new words to dictionary...\n",
      "New vocab size: 2195963\n",
      "Loading pre-trained embeddings for 1892506 words from data/glove.840B.300d.txt\n",
      "WARN: Duplicate embedding found for ����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������\n",
      "WARN: Duplicate embedding found for Decisión\n",
      "WARN: Duplicate embedding found for decisión\n",
      "WARN: Duplicate embedding found for Acórdão\n",
      "WARN: Duplicate embedding found for ̲̅̅\n",
      "WARN: Duplicate embedding found for brāhmaṇa\n",
      "WARN: Duplicate embedding found for Extensión\n",
      "WARN: Duplicate embedding found for Cível\n",
      "WARN: Duplicate embedding found for brāhmaṇas\n",
      "WARN: Duplicate embedding found for Vṛndāvana\n",
      "WARN: Duplicate embedding found for kΩ\n",
      "WARN: Duplicate embedding found for justiça\n",
      "WARN: Duplicate embedding found for acórdão\n",
      "WARN: Duplicate embedding found for Ṭhākura\n",
      "WARN: Duplicate embedding found for Resolución\n",
      "WARN: Duplicate embedding found for extensión\n",
      "WARN: Duplicate embedding found for ʹ\n",
      "WARN: Duplicate embedding found for Parīkṣit\n",
      "WARN: Duplicate embedding found for mΩ\n",
      "WARN: Duplicate embedding found for cível\n",
      "WARN: Duplicate embedding found for MΩ\n",
      "WARN: Duplicate embedding found for arrêté\n",
      "WARN: Duplicate embedding found for Tháng\n",
      "Loaded 1892506 embeddings (100.00%)\n",
      "Initializing tokenizer...\n"
     ]
    }
   ],
   "source": [
    "# %env CLASSPATH=corenlp/*\n",
    "from drqa.reader import Predictor\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# import os\n",
    "# os.environ['CLASSPATH'] = 'corenlp/*'\n",
    "\n",
    "qg1 = Predictor(model='ckpts/golden-retriever/hop1.mdl', tokenizer=None, embedding_file='data/glove.840B.300d.txt', num_workers=-1)\n",
    "qg1.cuda()\n",
    "# qg1.model.network.to(torch.device('cuda:0'))\n",
    "qg2 = Predictor(model='ckpts/golden-retriever/hop2.mdl', tokenizer=None, embedding_file='data/glove.840B.300d.txt', num_workers=-1)\n",
    "qg2.cuda()\n",
    "# qg2.model.network.to(torch.device('cuda:0'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "true"
   },
   "source": [
    "## Load dense retriever and query encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import faiss\n",
    "faiss.omp_set_num_threads(16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from argparse import Namespace\n",
    "\n",
    "args = Namespace(**{\n",
    "    \"model_name\": \"roberta-base\",\n",
    "    \"model_path\": \"ckpts/mdr/q_encoder.pt\",\n",
    "    \"index_prefix_path\": \"data/index/mdr/hotpot-paragraph-q-strict.hnsw\",\n",
    "    \"index_buffer_size\": 50000,\n",
    "    \"max_q_len\": 70,\n",
    "    \"max_q_sp_len\": 350\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-config.json from cache at /home/yuxiaoming/.cache/torch/transformers/e1a2a406b5a05063c31f4dfdee7608986ba7c6393f7f79db5e69dcd197208534.117c81977c5979de8c088352e74ec6e70f5c66096c28b61d3c50101609b39690\n",
      "Model config RobertaConfig {\n",
      "  \"architectures\": [\n",
      "    \"RobertaForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 514,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"vocab_size\": 50265\n",
      "}\n",
      "\n",
      "loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-config.json from cache at /home/yuxiaoming/.cache/torch/transformers/e1a2a406b5a05063c31f4dfdee7608986ba7c6393f7f79db5e69dcd197208534.117c81977c5979de8c088352e74ec6e70f5c66096c28b61d3c50101609b39690\n",
      "Model config RobertaConfig {\n",
      "  \"architectures\": [\n",
      "    \"RobertaForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 514,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"vocab_size\": 50265\n",
      "}\n",
      "\n",
      "loading file https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json from cache at /home/yuxiaoming/.cache/torch/transformers/d0c5776499adc1ded22493fae699da0971c1ee4c2587111707a4d177d20257a2.ef00af9e673c7160b4d41cfda1f48c5f4cba57d5142754525572a846a1ab1b9b\n",
      "loading file https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt from cache at /home/yuxiaoming/.cache/torch/transformers/b35e7cd126cd4229a746b5d5c29a749e8e84438b14bcdb575950584fe33207e8.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda\n",
      "loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-config.json from cache at /home/yuxiaoming/.cache/torch/transformers/e1a2a406b5a05063c31f4dfdee7608986ba7c6393f7f79db5e69dcd197208534.117c81977c5979de8c088352e74ec6e70f5c66096c28b61d3c50101609b39690\n",
      "Model config RobertaConfig {\n",
      "  \"architectures\": [\n",
      "    \"RobertaForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 514,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"vocab_size\": 50265\n",
      "}\n",
      "\n",
      "loading weights file https://cdn.huggingface.co/roberta-base-pytorch_model.bin from cache at /home/yuxiaoming/.cache/torch/transformers/80b4a484eddeb259bec2f06a6f2f05d90934111628e0e1c09a33bd4a121358e1.49b88ba7ec2c26a7558dda98ca3884c3b80fa31cf43a1b1f23aef3ff81ba344e\n",
      "All model checkpoint weights were used when initializing RobertaModel.\n",
      "\n",
      "All the weights of RobertaModel were initialized from the model checkpoint at roberta-base.\n",
      "If your task is similar to the task the model of the ckeckpoint was trained on, you can already use RobertaModel for predictions without further training.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RobertaCtxEncoder(\n",
       "  (encoder): RobertaModel(\n",
       "    (embeddings): RobertaEmbeddings(\n",
       "      (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",
       "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
       "      (token_type_embeddings): Embedding(1, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (project): Sequential(\n",
       "    (0): Linear(in_features=768, out_features=768, bias=True)\n",
       "    (1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import AutoConfig, AutoTokenizer\n",
    "from mdr.retrieval.models.retriever import RobertaCtxEncoder\n",
    "from utils.model_utils import load_state\n",
    "\n",
    "bert_config = AutoConfig.from_pretrained(args.model_name)\n",
    "tokenizer = AutoTokenizer.from_pretrained(args.model_name)\n",
    "query_encoder = RobertaCtxEncoder(bert_config, args)\n",
    "query_encoder = load_state(query_encoder, args.model_path, exact=False)\n",
    "device = torch.device('cuda:1')\n",
    "query_encoder.to(device)\n",
    "# if torch.cuda.device_count() > 1:\n",
    "#     query_encoder = torch.nn.DataParallel(query_encoder)\n",
    "query_encoder.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading index from data/index/mdr/hotpot-paragraph-q-strict.hnsw\n",
      "Loaded index of type <class 'faiss.swigfaiss.IndexHNSWFlat'> and size 5232080\n"
     ]
    }
   ],
   "source": [
    "from dense_indexers import DenseHNSWFlatIndexer, DenseFlatIndexer\n",
    "\n",
    "vector_size = bert_config.hidden_size\n",
    "dense_indexer = DenseHNSWFlatIndexer(vector_size, args.index_buffer_size)\n",
    "dense_indexer.deserialize_from(args.index_prefix_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from retriever import DenseRetriever\n",
    "\n",
    "dense_retriever = DenseRetriever(dense_indexer, query_encoder, tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "true"
   },
   "source": [
    "## Load corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loaded 5232080 passages from data/corpus/hotpot-paragraph-5.tsv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5232080\n",
      "5232080\n"
     ]
    }
   ],
   "source": [
    "from utils.data_utils import load_corpus\n",
    "\n",
    "corpus, title2id = load_corpus('data/corpus/hotpot-paragraph-5.tsv', for_hotpot=True, require_hyperlinks=True)\n",
    "print(len(corpus))\n",
    "print(len(title2id))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "true"
   },
   "source": [
    "## Generate greedy step transition data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_global_logging_level(logging.WARNING, [\"elasticsearch\"])\n",
    "logger.setLevel(logging.WARNING)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import redis\n",
    "\n",
    "query_redis = redis.Redis(host='10.60.1.79', port=6379, db=2, password='redis4zyc', decode_responses=True)\n",
    "bm25_redis = redis.Redis(host='10.60.1.79', port=6379, db=3, password='redis4zyc', decode_responses=True)  # 0\n",
    "mdr_redis = redis.Redis(host='10.60.1.79', port=6379, db=4, password='redis4zyc', decode_responses=True)  # 1"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "RET_SIZE = 1000\n",
    "hits_list1 = dense_retriever.msearch(questions, max(RET_SIZE, 1000), args.max_q_len, 128)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "for question, hits_list in zip(questions, hits_list1):\n",
    "    if not mdr_redis.exists(question) or (mdr_redis.llen(question) < RET_SIZE and mdr_redis.lindex(question, -1) != 'EOL'):\n",
    "        hits = hits_list[0]\n",
    "        mdr_redis.delete(question)\n",
    "        if len(hits) < RET_SIZE:\n",
    "            mdr_redis.rpush(question, *(hits + ['EOL']))\n",
    "        else:\n",
    "            mdr_redis.rpush(question, *hits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9514fe4e2fac4a6190842f3e9852997e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=90447.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5adf8d5b5542995534e8c7e1: Unable recall SP2('2016 Marrakesh ePrix') through the first 1000 retrieval(Q + 'Sébastien Buemi') results\n",
      "5abbbd0f55429931dba144d5: Unable recall SP2('Hot Feet') through the first 1000 retrieval(Q + 'Maurice Hines') results\n",
      "5ab1ce395542997061209579: Unable recall SP2('The Adventures of Brer Rabbit') through the first 1000 retrieval(Q + \"Br'er Rabbit\") results\n",
      "5ac2c958554299657fa2905b: Unable recall SP2('Be Right There') through the first 1000 retrieval(Q + 'The Warriors Gate') results\n",
      "5ae1130a5542997b2ef7d0d3: Unable recall SP2('John Wallace Crawford') through the first 1000 retrieval(Q + 'American Horse (elder)') results\n",
      "5a80271a5542992e7d278dfa: Unable recall SP2(\"Minnesota's 10th congressional district\") through the first 1000 retrieval(Q + \"Minnesota's 3rd congressional district\") results\n",
      "5a77732b55429967ab105183: Unable recall SP2('Le piccole storie') through the first 1000 retrieval(Q + 'Julius Caesar (play)') results\n",
      "5a879c8e5542994846c1cdb3: Unable recall SP2('Skogsrå') through the first 1000 retrieval(Q + 'Hulder') results\n",
      "5abb10ac5542992ccd8e7ef5: Unable recall SP2('Ashley Hicks') through the first 1000 retrieval(Q + 'Lady Pamela Hicks') results\n",
      "5abd42a355429924427fcf54: Unable recall SP2('Buddy Bell') through the first 1000 retrieval(Q + 'David Bell (baseball)') results\n",
      "5a81a6c45542990a1d231e48: Unable recall SP2('2752 Wu Chien-Shiung') through the first 1000 retrieval(Q + 'Chien-Shiung Wu') results\n",
      "5ab86a2355429934fafe6dae: Unable recall SP2('68th World Science Fiction Convention') through the first 1000 retrieval(Q + 'Melbourne Convention and Exhibition Centre') results\n",
      "5ae3ff115542996836b02c09: Unable recall SP2('Vs. Tour') through the first 1000 retrieval(Q + 'Vs. (Pearl Jam album)') results\n",
      "5a7716ac55429972597f142e: Unable recall SP2('Rose Batiste') through the first 1000 retrieval(Q + 'Blackpool Mecca') results\n",
      "5a81936d554299260e20a29f: Unable recall SP2('Oslo, August 31st') through the first 1000 retrieval(Q + 'Joachim Trier') results\n",
      "5a90538d5542995651fb5144: Unable recall SP2('Maeve') through the first 1000 retrieval(Q + 'Queen Mab') results\n",
      "5a75dc4a554299109176e5a9: Unable recall SP2('Anne Maddocks') through the first 1000 retrieval(Q + 'Francis Poulenc') results\n",
      "5ac0754c554299012d1db5f5: Unable recall SP2('Ostinato') through the first 1000 retrieval(Q + 'Bitter Sweet Symphony') results\n",
      "5abd13e455429933744ab70d: Unable recall SP2('Valie Export') through the first 1000 retrieval(Q + 'Gary Sherman (director)') results\n",
      "5a7cf32155429909bec768bc: Unable recall SP2('Jane the Virgin (season 1)') through the first 1000 retrieval(Q + 'Gina Rodriguez') results\n",
      "5a81253b55429926c1cdacf5: Unable recall SP2('C.I.D. (1990 film)') through the first 1000 retrieval(Q + 'Witness (1985 film)') results\n",
      "5a74852655429974ef308c18: Unable recall SP2('Holiday Inn (musical)') through the first 1000 retrieval(Q + 'Holiday Inn (film)') results\n",
      "5a7ded595542995f4f402313: Unable recall SP2('Louis Joseph, Dauphin of France') through the first 1000 retrieval(Q + 'Marie Antoinette') results\n",
      "5a81e5945542995ce29dcc6a: Unable recall SP2('166th Air Refueling Squadron') through the first 1000 retrieval(Q + 'Rickenbacker Air National Guard Base') results\n",
      "5add5ef85542992ae4cec52d: Unable recall SP2('Donnington Bridge Road') through the first 1000 retrieval(Q + 'Oxford') results\n",
      "5ac1b08155429963665198ef: Unable recall SP2('Ben Houge') through the first 1000 retrieval(Q + 'St. Olaf College') results\n",
      "5a8604cd554299211dda2a38: Unable recall SP2('Four-minute mile') through the first 1000 retrieval(Q + 'Roger Bannister') results\n",
      "5a846dae5542996488c2e543: Unable recall SP2('Pajama Party (group)') through the first 1000 retrieval(Q + 'Daphne Rubin-Vega') results\n",
      "5ab2c9d1554299340b525550: Unable recall SP2('Celtic Woman: Voices of Angels') through the first 1000 retrieval(Q + 'Susan McFadden') results\n",
      "5a8cc89a554299653c1aa122: Unable recall SP2('You Will Only Break My Heart') through the first 1000 retrieval(Q + 'Delta Goodrem') results\n",
      "5add3f215542997545bbbcfb: Unable recall SP2('Beyond This Place (1957 TV show)') through the first 1000 retrieval(Q + 'Sidney Lumet') results\n",
      "5a77881f5542997042120a96: Unable recall SP2('Louisville Gardens') through the first 1000 retrieval(Q + 'Jerry Abramson') results\n",
      "5ac2c983554299657fa2905c: Unable recall SP2('Plimpton Sieve Portrait of Queen Elizabeth I') through the first 1000 retrieval(Q + 'Folger Shakespeare Library') results\n",
      "5abb06895542996cc5e49f50: Unable recall SP2('Tom Cross (film editor)') through the first 1000 retrieval(Q + 'Damien Chazelle') results\n",
      "5ac1aea755429963665198e8: Unable recall SP2('The Way We Live Now (2001 TV serial)') through the first 1000 retrieval(Q + 'Anthony Trollope') results\n",
      "5a8b4a7555429971feec469e: Unable recall SP2('Gogebic Taconite') through the first 1000 retrieval(Q + 'Chris Cline') results\n",
      "5ae677cd5542996d980e7b6d: Unable recall SP2('The Ascendicate') through the first 1000 retrieval(Q + 'Alien Ant Farm') results\n",
      "5ae6c0625542995703ce8b99: Unable recall SP2('Benjamin Champney') through the first 1000 retrieval(Q + 'White Mountain art') results\n",
      "5ab72a5d55429928e1fe3826: Unable recall SP2('Il mostro di Firenze (miniseries)') through the first 1000 retrieval(Q + 'Monster of Florence') results\n",
      "5a7c3ebd55429935c91b513f: Unable recall SP2('Old Grey Whistle Theft') through the first 1000 retrieval(Q + 'Father Ted') results\n",
      "5ae3e5a25542996836b02bea: Unable recall SP2('Sara McGreavy') through the first 1000 retrieval(Q + 'Cross country running') results\n",
      "5a8cd4b8554299441c6b9f19: Unable recall SP2('Cimarron Strip') through the first 1000 retrieval(Q + 'Lawrence of Arabia (film)') results\n",
      "5ae657735542995703ce8b59: Unable recall SP2('Tony M.') through the first 1000 retrieval(Q + 'The New Power Generation') results\n",
      "5a9046c155429933b8a204c5: Unable recall SP2('Antique (film)') through the first 1000 retrieval(Q + 'Antique Bakery') results\n",
      "5abf09ec5542997719eab62e: Unable recall SP2('The Flower Drum Song') through the first 1000 retrieval(Q + 'James Shigeta') results\n",
      "5adff9ff554299603e4183dc: Unable recall SP2('Stutter edit') through the first 1000 retrieval(Q + \"Drop It Like It's Hot\") results\n",
      "5ae5d7455542996de7b71a2f: Unable recall SP2('Harvest Moon (album)') through the first 1000 retrieval(Q + 'Harvest (Neil Young album)') results\n",
      "5ac117f05542992a796dedd8: Unable recall SP2('Firebird (Lackey novel)') through the first 1000 retrieval(Q + 'The Firebird') results\n",
      "5ac5174355429924173fb59f: Unable recall SP2('Race and ethnicity in the NHL') through the first 1000 retrieval(Q + 'Evander Kane') results\n",
      "5ae7411c5542997b22f6a63f: Unable recall SP2('South Tongu (Ghana parliament constituency)') through the first 1000 retrieval(Q + 'South Tongu District') results\n",
      "5abb69f25542996cc5e49fea: Unable recall SP2('Hospital Argerich') through the first 1000 retrieval(Q + 'La Boca') results\n",
      "5a88ed85554299206df2b3b4: Unable recall SP2('San Casimiro') through the first 1000 retrieval(Q + 'Saint Casimir') results\n",
      "5a8b242d55429950cd6afc55: Unable recall SP2('George Frederick, Margrave of Brandenburg-Ansbach') through the first 1000 retrieval(Q + 'George, Margrave of Brandenburg-Ansbach') results\n",
      "5a7da83e5542995f4f4022a8: Unable recall SP2('Order of Saint Anastasia') through the first 1000 retrieval(Q + 'Maria Vladimirovna, Grand Duchess of Russia') results\n",
      "5a7d0e1c554299683c1c6422: Unable recall SP2('Shuuto') through the first 1000 retrieval(Q + 'Daisuke Matsuzaka') results\n",
      "5aba8e395542994dbf019940: Unable recall SP2('Leonard Bishop') through the first 1000 retrieval(Q + 'Joseph Heller') results\n",
      "5ab3d9ec55429969a97a81d4: Unable recall SP2('Sports in Washington, D.C.') through the first 1000 retrieval(Q + 'Super Bowl XXVI') results\n",
      "5ae0fbd5554299422ee9959b: Unable recall SP2('János Kovács') through the first 1000 retrieval(Q + 'Lincoln City F.C.') results\n",
      "5ae2b2765542994d89d5b433: Unable recall SP2('Federico Tesio Stakes top three finishers') through the first 1000 retrieval(Q + 'Laurel, Maryland') results\n",
      "5abb2e505542996cc5e49fd0: Unable recall SP2('Benjamin Helm House') through the first 1000 retrieval(Q + 'Benjamin Hardin Helm') results\n",
      "5a740bc25542993e948e87b3: Unable recall SP2('Margie Joseph') through the first 1000 retrieval(Q + 'Paul McCartney') results\n",
      "5ab961f95542996be20204b8: Unable recall SP2('Sliding filament theory') through the first 1000 retrieval(Q + 'Andrew Huxley') results\n",
      "5ab850ec55429934fafe6d58: Unable recall SP2('Dream Theater discography') through the first 1000 retrieval(Q + 'John Petrucci') results\n",
      "5a7cdcbd554299683c1c63a7: Unable recall SP2('John Frusciante discography') through the first 1000 retrieval(Q + 'Californication (album)') results\n",
      "5ac11988554299012d1db699: Unable recall SP2('Ostenk') through the first 1000 retrieval(Q + 'Municipality') results\n",
      "5ade674a5542992fa25da770: Unable recall SP2('Miley Cyrus videography') through the first 1000 retrieval(Q + 'Liam Hemsworth') results\n",
      "5ae82b1d5542997ec272773d: Unable recall SP2('Troller Pantanal') through the first 1000 retrieval(Q + 'Troller Veículos Especiais') results\n",
      "5adcdcd655429947343537b4: Unable recall SP2('Hemangada') through the first 1000 retrieval(Q + 'Kālidāsa') results\n",
      "5abbd225554299660624172d: Unable recall SP2('Gangs of New York') through the first 1000 retrieval(Q + 'Miramax') results\n",
      "5adbd9fc5542996e6852525b: Unable recall SP2('Gloria (TV series)') through the first 1000 retrieval(Q + 'Sally Struthers') results\n",
      "5a7e1b585542995ed0d16699: Unable recall SP2('Yusupov Palace (Crimea)') through the first 1000 retrieval(Q + 'Felix Yusupov') results\n",
      "5a7639395542994ccc918724: Unable recall SP2('Moonrunners') through the first 1000 retrieval(Q + 'Thunder Road (film)') results\n",
      "5a77caca55429967ab1052a5: Unable recall SP2('2014 Arizona Diamondbacks season') through the first 1000 retrieval(Q + 'Alan Trammell') results\n",
      "5abd316d5542992ac4f3821c: Unable recall SP2('Bristol (CDP), New Hampshire') through the first 1000 retrieval(Q + 'Bristol, New Hampshire') results\n",
      "5adf55665542995534e8c793: Unable recall SP2('I Wanna Fall in Love') through the first 1000 retrieval(Q + 'Wicked Game') results\n",
      "5a74234155429974ef308bb3: Unable recall SP2('Infinity Cat Recordings') through the first 1000 retrieval(Q + 'Grand Ole Opry') results\n",
      "5abffa615542994516f45567: Unable recall SP2('Ralph K. Pedersen') through the first 1000 retrieval(Q + 'Levittown, New York') results\n",
      "5a79dbdd5542994bb94570f7: Unable recall SP2(\"1921–22 Connecticut Aggies men's basketball team\") through the first 1000 retrieval(Q + 'University of Connecticut') results\n",
      "5ae731ff5542991e8301cbc0: Unable recall SP2('Faraday (disambiguation)') through the first 1000 retrieval(Q + 'Michael Faraday') results\n",
      "5ae5cde95542992663a4f228: Unable recall SP2('Capital punishment in Canada') through the first 1000 retrieval(Q + 'Ronald Turpin') results\n",
      "5a82351e5542995ce29dccd8: Unable recall SP2('Kwabre West (Ghana parliament constituency)') through the first 1000 retrieval(Q + 'Kwabre District') results\n",
      "5add3e405542992c1e3a257c: Unable recall SP2('Scott Alden') through the first 1000 retrieval(Q + 'John Alden') results\n",
      "5ab1f91a55429970612095a2: Unable recall SP2('Shepherd of Fire') through the first 1000 retrieval(Q + 'EA Sports UFC') results\n",
      "5adbf9d855429947ff173882: Unable recall SP2('East Springfield Union School') through the first 1000 retrieval(Q + 'Concrete masonry unit') results\n",
      "5ab8bd1355429919ba4e2366: Unable recall SP2('Brahmalokam To Yamalokam Via Bhulokam') through the first 1000 retrieval(Q + 'M. M. Srilekha') results\n",
      "5ac425215542997ea680ca11: Unable recall SP2('1950–51 Chester F.C. season') through the first 1000 retrieval(Q + 'Chester City F.C.') results\n",
      "5a79e1115542994bb945711b: Unable recall SP2('Wallace, Kansas') through the first 1000 retrieval(Q + 'Wallace County, Kansas') results\n",
      "5ae72d875542991e8301cb9e: Unable recall SP2('Xavier Bertrand') through the first 1000 retrieval(Q + 'Nicolas Sarkozy') results\n",
      "5a8c91995542995e66a47625: Unable recall SP2('Switched-On Bach II') through the first 1000 retrieval(Q + 'Wendy Carlos') results\n",
      "5a8c4b0d5542995e66a4759d: Unable recall SP2('Erato (dryad)') through the first 1000 retrieval(Q + 'Callisto (mythology)') results\n",
      "5ae5f2265542996de7b71a77: Unable recall SP2('Massachusetts General Hospital, Bulfinch Building') through the first 1000 retrieval(Q + 'Diethyl ether') results\n",
      "5abd30225542993062266bc7: Unable recall SP2('IPC Shopping Centre') through the first 1000 retrieval(Q + 'The Curve (shopping mall)') results\n",
      "5a7b76135542997c3ec9719b: Unable recall SP2('Blue Moon Rising (film)') through the first 1000 retrieval(Q + '2009–10 Manchester City F.C. season') results\n",
      "5abd1d5d5542992ac4f381e4: Unable recall SP2('Avril Coleridge-Taylor') through the first 1000 retrieval(Q + 'Samuel Coleridge-Taylor') results\n",
      "5adda1b55542997dc790700a: Unable recall SP2('M. M. Kalburgi') through the first 1000 retrieval(Q + 'Hampi') results\n",
      "5ab706915542991d3222375d: Unable recall SP2('Good Luck!') through the first 1000 retrieval(Q + 'Just My Luck (2006 film)') results\n",
      "5ac4f2525542996feb3fe996: Unable recall SP2('The Groop (US band)') through the first 1000 retrieval(Q + 'Midnight Cowboy') results\n",
      "5ac31700554299741d48a1c6: Unable recall SP2('Linda Reisman') through the first 1000 retrieval(Q + 'Assassination Tango') results\n",
      "5a730b645542991f29ee2cfb: Unable recall SP2('UFC Fight Night: Holm vs. Correia') through the first 1000 retrieval(Q + 'Ultimate Fighting Championship') results\n",
      "5add5b4c5542997545bbbd39: Unable recall SP2(\"Eastern Michigan Eagles men's basketball\") through the first 1000 retrieval(Q + 'Mid-American Conference') results\n",
      "5ac4d76e554299076e296e10: Unable recall SP2('Plop Goes the Weasel') through the first 1000 retrieval(Q + 'Pop Goes the Weasel') results\n",
      "5a77ad3955429949eeb29f2c: Unable recall SP2('Argyroeides tricolor') through the first 1000 retrieval(Q + 'Amazons') results\n",
      "5a8106e7554299260e20a208: Unable recall SP2('Alles Tot') through the first 1000 retrieval(Q + 'Iron Maiden') results\n",
      "5ab5eb89554299494045f0b7: Unable recall SP2('2014 FIFA World Cup Final') through the first 1000 retrieval(Q + 'Mario Götze') results\n",
      "5adc32d85542994650320cae: Unable recall SP2('Hackensack, Minnesota') through the first 1000 retrieval(Q + 'Paul Bunyan') results\n",
      "5ae3db125542996836b02bdc: Unable recall SP2('Lync') through the first 1000 retrieval(Q + 'Olympia, Washington') results\n",
      "5a7c648d5542996dd594b912: Unable recall SP2('50 to 1') through the first 1000 retrieval(Q + 'Calvin Borel') results\n",
      "5a8c2d67554299240d9c20ba: Unable recall SP2('George Wickham') through the first 1000 retrieval(Q + 'Problem gambling') results\n",
      "5abbc1a85542992ccd8e7f88: Unable recall SP2('Marengo (horse)') through the first 1000 retrieval(Q + 'Battle of Marengo') results\n",
      "5adfc5ed55429942ec259b46: Unable recall SP2('Freudian (album)') through the first 1000 retrieval(Q + 'BadBadNotGood') results\n",
      "5a8f144a5542995b44241f57: Unable recall SP2('Brit Pack (actors)') through the first 1000 retrieval(Q + 'Brat Pack (actors)') results\n",
      "5a74a05e55429929fddd8487: Unable recall SP2('Devante Bond') through the first 1000 retrieval(Q + 'Oklahoma Sooners football') results\n",
      "5add32b45542994ed6169ccf: Unable recall SP2('Itea (plant)') through the first 1000 retrieval(Q + 'Alocasia') results\n",
      "5ac1757a5542994d76dcce42: Unable recall SP2('Haber (film)') through the first 1000 retrieval(Q + 'Fritz Haber') results\n",
      "5a7d33df554299452d57bb54: Unable recall SP2('Bluetown, Texas') through the first 1000 retrieval(Q + 'Brownsville, Texas') results\n",
      "5a8bf52d5542997f31a41df6: Unable recall SP2('Women in Medieval Scotland') through the first 1000 retrieval(Q + 'Margaret, Maid of Norway') results\n",
      "5a84796f554299123d8c2262: Unable recall SP2('Ugo Falena') through the first 1000 retrieval(Q + 'Franco Alfano') results\n",
      "5ab2994d554299449642c924: Unable recall SP2('Tell Me You Get Lonely') through the first 1000 retrieval(Q + 'Frankie Ballard') results\n",
      "5ae13b1a554299422ee99631: Unable recall SP2('Kris Tompkins') through the first 1000 retrieval(Q + 'Douglas Tompkins') results\n",
      "5a8075c7554299485f5985f9: Unable recall SP2('WFPC-LP') through the first 1000 retrieval(Q + 'Marlin Fitzwater') results\n",
      "5abdb85d5542993f32c2a013: Unable recall SP2('Arya (name)') through the first 1000 retrieval(Q + 'Arya Stark') results\n",
      "5abaadf855429901930fa893: Unable recall SP2('I Know What I Like') through the first 1000 retrieval(Q + 'Ronnie Lott') results\n",
      "5a7926fa554299148911fa0f: Unable recall SP2('Scott Marr') through the first 1000 retrieval(Q + \"Albany Great Danes men's lacrosse\") results\n",
      "5a7f62a95542994857a76718: Unable recall SP2(\"Gavin O'Connor (director)\") through the first 1000 retrieval(Q + 'Suicide Squad (film)') results\n",
      "5a8c7d1d5542995e66a4761d: Unable recall SP2('1990–91 Washington Bullets season') through the first 1000 retrieval(Q + 'Harvey Grant') results\n",
      "5ac2a27a554299657fa28ff4: Unable recall SP2('Ahafo Ano South (Ghana parliament constituency)') through the first 1000 retrieval(Q + 'Ahafo Ano South District') results\n",
      "5a74e39b55429916b0164235: Unable recall SP2('Aiden Flowers') through the first 1000 retrieval(Q + 'Nate Parker') results\n",
      "5ae347925542992e3233c365: Unable recall SP2('Levi-Civita symbol') through the first 1000 retrieval(Q + 'Tullio Levi-Civita') results\n",
      "5a78d00f554299029c4b5ea5: Unable recall SP2('The Very Best of Macy Gray') through the first 1000 retrieval(Q + 'Walk This Way') results\n",
      "5ac0e704554299294b219050: Unable recall SP2('Marvin Cohen (American writer)') through the first 1000 retrieval(Q + 'Shakespeare in the Park (New York City)') results\n",
      "5a83aeb95542996488c2e486: Unable recall SP2('Jorge Zorreguieta') through the first 1000 retrieval(Q + 'Queen Máxima of the Netherlands') results\n",
      "5ab6e536554299110f219a74: Unable recall SP2('Punch and the Judy') through the first 1000 retrieval(Q + 'Edward Gordon Craig') results\n",
      "5ab286ea554299449642c8e3: Unable recall SP2('Patty Dann') through the first 1000 retrieval(Q + 'Cher') results\n",
      "5a7e4b0d554299495941998d: Unable recall SP2('Concert at the End of Summer') through the first 1000 retrieval(Q + 'Antonín Dvořák') results\n",
      "5a7327cc5542994cef4bc482: Unable recall SP2('Kabhie Kabhie (1976 film)') through the first 1000 retrieval(Q + 'Mukesh (singer)') results\n",
      "5a8ca7fe554299240d9c21ba: Unable recall SP2('Jorge Zorreguieta') through the first 1000 retrieval(Q + 'Queen Máxima of the Netherlands') results\n",
      "5abb11265542992ccd8e7ef6: Unable recall SP2('Eka Darville') through the first 1000 retrieval(Q + 'Afro-Jamaican') results\n",
      "5a88a8e75542997e5c09a697: Unable recall SP2('The Canadas') through the first 1000 retrieval(Q + 'British colonization of the Americas') results\n",
      "5a8efd1655429918e830d178: Unable recall SP2('Sink the Belgrano!') through the first 1000 retrieval(Q + 'ARA General Belgrano') results\n",
      "5a87433055429960ec39b77e: Unable recall SP2('Godric (novel)') through the first 1000 retrieval(Q + 'Godric of Finchale') results\n",
      "5ae502e355429960a22e0239: Unable recall SP2('Rani Varma') through the first 1000 retrieval(Q + 'Manik Varma') results\n",
      "5a85d0045542997175ce2039: Unable recall SP2('Dull Boy') through the first 1000 retrieval(Q + 'Mudvayne') results\n",
      "5a87dcb95542997e5c09a57e: Unable recall SP2('Un gancho al corazón') through the first 1000 retrieval(Q + 'You Are the One (Argentine TV series)') results\n",
      "5a802bfd554299485f59857d: Unable recall SP2('Look Back in Anger (TV series)') through the first 1000 retrieval(Q + 'Bae Doona') results\n",
      "5ae744c65542991e8301cc4d: Unable recall SP2('Richard G. Gotti') through the first 1000 retrieval(Q + 'John Gotti') results\n",
      "5abcfe34554299114383a1bd: Unable recall SP2('The Duel (2016 film)') through the first 1000 retrieval(Q + 'Liam Hemsworth') results\n",
      "5ac2c9de554299657fa29060: Unable recall SP2('JC Aragone') through the first 1000 retrieval(Q + '2017 US Open (tennis)') results\n",
      "5a7351875542994cef4bc53a: Unable recall SP2('Yosgart Gutiérrez') through the first 1000 retrieval(Q + 'Óscar Pérez Rojas') results\n",
      "5ae7871e5542994a481bbd5d: Unable recall SP2('ENnies') through the first 1000 retrieval(Q + 'EN World') results\n",
      "5ae0efc55542990adbacf6e1: Unable recall SP2('12th Lumières Awards') through the first 1000 retrieval(Q + 'Tell No One') results\n",
      "5a7bc7b2554299294a54aacd: Unable recall SP2('Chiral symmetry breaking') through the first 1000 retrieval(Q + 'Yoichiro Nambu') results\n",
      "5a74c2c85542996c70cfaddc: Unable recall SP2('Kermit Unpigged') through the first 1000 retrieval(Q + 'Miss Piggy') results\n",
      "5a8e8a3d5542995085b37447: Unable recall SP2('Mercedes-Benz W25') through the first 1000 retrieval(Q + 'Mercedes-Benz W125') results\n",
      "5ae225d2554299234fd043f7: Unable recall SP2('1906 United Kingdom heat wave') through the first 1000 retrieval(Q + 'Gordon Castle') results\n",
      "5ae112075542990adbacf711: Unable recall SP2('Mark Anthony Neal') through the first 1000 retrieval(Q + 'Duke University') results\n",
      "5a8dbd20554299441c6ba07c: Unable recall SP2('Plantation High School') through the first 1000 retrieval(Q + 'Colonel Reb') results\n",
      "5a7928fa554299148911fa11: Unable recall SP2('Big Four (Western Europe)') through the first 1000 retrieval(Q + 'Group of Ten (economics)') results\n",
      "5a8655bf554299211dda2af8: Unable recall SP2('The Jane Pickens Show') through the first 1000 retrieval(Q + 'Eddie Fisher (singer)') results\n",
      "Unable recall SP1 through the first 1000 retrieval results: 5ae556dd55429908b63265e6\n",
      "5a812472554299260e20a25b: Unable recall SP2('Michele Landis Dauber') through the first 1000 retrieval(Q + 'People v. Turner') results\n",
      "5abe39005542993f32c2a09a: Unable recall SP2('Sarah Jeong') through the first 1000 retrieval(Q + 'Vice (magazine)') results\n",
      "5ab2bd765542991669774084: Unable recall SP2('Rehearsal Club (London)') through the first 1000 retrieval(Q + 'Prince Christian of Schleswig-Holstein') results\n",
      "5a8f813c5542992414482aea: Unable recall SP2('Evergreen International Aviation') through the first 1000 retrieval(Q + 'Evergreen International Airlines') results\n",
      "5a8a2fb955429930ff3c0ce8: Unable recall SP2('Mark Livolsi') through the first 1000 retrieval(Q + 'The Devil Wears Prada (film)') results\n",
      "5adf46265542992d7e9f9318: Unable recall SP2('Blanvalet') through the first 1000 retrieval(Q + 'Karin Slaughter') results\n",
      "Unable recall SP1 through the first 1000 retrieval results: 5a87d2db5542994775f607f9\n",
      "5a7651375542992d0ec060a2: Unable recall SP2('Cape Girardeau Regional Airport') through the first 1000 retrieval(Q + 'Cape Air') results\n",
      "5ae32bbe55429928c4239645: Unable recall SP2('Canada Bay, New South Wales') through the first 1000 retrieval(Q + 'City of Canada Bay') results\n",
      "5ab6db70554299110f219a6b: Unable recall SP2(\"The Rock 'n' Rave Infection\") through the first 1000 retrieval(Q + 'Lance Hoyt') results\n",
      "5adcdaf75542990d50227d47: Unable recall SP2('Under the Influence (compilation album)') through the first 1000 retrieval(Q + 'Ian Brown') results\n",
      "Unable recall SP1 through the first 1000 retrieval results: 5ae3edeb5542995ad6573ca2\n",
      "5a904f755542995651fb5124: Unable recall SP2('Felix Landau') through the first 1000 retrieval(Q + 'Maria Altmann') results\n",
      "5a8ba2215542995d1e6f142a: Unable recall SP2('Kevin Daniels') through the first 1000 retrieval(Q + 'Ladder 49') results\n",
      "5ae0e44e5542990adbacf6b5: Unable recall SP2('Forgotten Coast') through the first 1000 retrieval(Q + 'Tyndall Air Force Base') results\n",
      "5ac01c3d5542992a796dec8f: Unable recall SP2('Divided by Night') through the first 1000 retrieval(Q + 'The Crystal Method') results\n",
      "5abb7f675542993f40c73b15: Unable recall SP2('Susan May Williams') through the first 1000 retrieval(Q + 'Erie Canal') results\n",
      "5ab2f5ab55429929539468d0: Unable recall SP2('Robert Maxson') through the first 1000 retrieval(Q + 'Incline Village, Nevada') results\n",
      "5ab7e6b0554299366779403e: Unable recall SP2('Cat Girl') through the first 1000 retrieval(Q + 'Cat People (1942 film)') results\n",
      "5ae213bd554299495565d9e5: Unable recall SP2('Rock Bottom (SpongeBob SquarePants)') through the first 1000 retrieval(Q + 'SpongeBob SquarePants (season 1)') results\n",
      "5ac3e4675542997ea680c983: Unable recall SP2('Eleanor Bull') through the first 1000 retrieval(Q + 'Christopher Marlowe') results\n",
      "5ab34b0b55429969a97a8133: Unable recall SP2('7th Air Refueling Squadron') through the first 1000 retrieval(Q + 'Robins Air Force Base') results\n",
      "5a89db4b5542992e4fca83f3: Unable recall SP2('Call of the Wild (disambiguation)') through the first 1000 retrieval(Q + 'The Call of the Wild') results\n",
      "5a8edaef55429917b4a5bde6: Unable recall SP2('The Michael J. Fox Show') through the first 1000 retrieval(Q + 'Spin City') results\n",
      "5ae694a65542991bbc976142: Unable recall SP2('Bondage Goat Zombie') through the first 1000 retrieval(Q + 'Belphegor (band)') results\n",
      "5a86f9145542991e7718169d: Unable recall SP2('Great Northern 325') through the first 1000 retrieval(Q + 'Great Northern Railway (U.S.)') results\n",
      "5adf3f4d5542993a75d26449: Unable recall SP2('Flava in Ya Ear') through the first 1000 retrieval(Q + 'Craig Mack') results\n",
      "5adf1bd85542995ec70e8f76: Unable recall SP2('James H. Doolittle Award') through the first 1000 retrieval(Q + 'Jimmy Doolittle') results\n",
      "5a7b19295542992d025e676a: Unable recall SP2('Shanagarry') through the first 1000 retrieval(Q + 'Rachel Allen') results\n",
      "5a82868555429954d2e2eb62: Unable recall SP2('Bob Mackie') through the first 1000 retrieval(Q + 'Carol Burnett') results\n",
      "5ae3fc255542995dadf24286: Unable recall SP2('Specter of the Rose') through the first 1000 retrieval(Q + 'Tamara Geva') results\n",
      "5adf079a5542995534e8c737: Unable recall SP2('The Compleat Beatles') through the first 1000 retrieval(Q + 'The Compleat Angler') results\n",
      "5a86e8f25542991e77181686: Unable recall SP2('Aqua Teen Hunger Force (season 4)') through the first 1000 retrieval(Q + 'South Jersey') results\n",
      "5ae711c1554299572ea546b3: Unable recall SP2('Justin Kiriakis and Adrienne Johnson') through the first 1000 retrieval(Q + 'Wally Kurth') results\n",
      "5ab91f23554299753720f70b: Unable recall SP2('Concerto for Two Pianos and Orchestra (Poulenc)') through the first 1000 retrieval(Q + 'Pavane pour une infante défunte') results\n",
      "5a77c55f5542995d831812bb: Unable recall SP2('Luminary (astrology)') through the first 1000 retrieval(Q + 'Aristotle') results\n",
      "5ac2fc9155429921a00ab0bf: Unable recall SP2('Master Tracks') through the first 1000 retrieval(Q + 'Moe Berg') results\n",
      "5a8f989b55429918e830d28b: Unable recall SP2('Branko Tomović') through the first 1000 retrieval(Q + 'Charlotte Riley') results\n",
      "5a89086e554299515336126d: Unable recall SP2('Unfold (Marié Digby album)') through the first 1000 retrieval(Q + 'Umbrella (song)') results\n",
      "5a8b051f55429949d91db468: Unable recall SP2('Zan Perrion') through the first 1000 retrieval(Q + 'Adam Rodriguez') results\n",
      "5adc8d965542994734353740: Unable recall SP2('Josiah Cotton') through the first 1000 retrieval(Q + 'Cotton Mather') results\n",
      "5ac15d0d5542994ab5c67cfc: Unable recall SP2('DeMarre Carroll') through the first 1000 retrieval(Q + 'Brooklyn Nets') results\n",
      "5a75b0a15542992d0ec05fe2: Unable recall SP2(\"Hund's cases\") through the first 1000 retrieval(Q + 'Friedrich Hund') results\n",
      "5a838732554299123d8c214a: Unable recall SP2('James S. Harlan') through the first 1000 retrieval(Q + 'John Marshall Harlan') results\n",
      "5ae7c4375542993210983f3e: Unable recall SP2('The Promise (2010 play)') through the first 1000 retrieval(Q + 'H. H. Asquith') results\n",
      "5a7adea655429931da12c973: Unable recall SP2('Bobby Bitch') through the first 1000 retrieval(Q + 'Bobby Shmurda') results\n",
      "5a8c4e6d554299653c1aa033: Unable recall SP2(\"Gavin O'Connor (director)\") through the first 1000 retrieval(Q + 'Miracle (2004 film)') results\n",
      "5a79b2b5554299148911fa38: Unable recall SP2('Oriental Star Agencies') through the first 1000 retrieval(Q + 'Malkit Singh') results\n",
      "5a7b8d67554299294a54a9ed: Unable recall SP2('Johann Georg Mozart') through the first 1000 retrieval(Q + 'Leopold Mozart') results\n",
      "5ae14f9b5542990adbacf747: Unable recall SP2('Solar eclipse of February 26, 1998') through the first 1000 retrieval(Q + 'Curaçao') results\n",
      "5abf937f5542993fe9a41e08: Unable recall SP2('The Revenge of Seven') through the first 1000 retrieval(Q + 'Lorien Legacies') results\n",
      "5abee9f05542997719eab5f6: Unable recall SP2('Tumbledown (2015 film)') through the first 1000 retrieval(Q + 'Joe Manganiello') results\n",
      "5ab61777554299488d4d9aa9: Unable recall SP2('Jim Danforth') through the first 1000 retrieval(Q + 'One Million Years B.C.') results\n",
      "5ab8186d55429916710eafec: Unable recall SP2('Master Raro') through the first 1000 retrieval(Q + 'Clara Schumann') results\n",
      "5a761be955429976ec32bd0f: Unable recall SP2('National Motor Museum Monorail') through the first 1000 retrieval(Q + 'Butlins') results\n",
      "5a89e348554299669944a5f6: Unable recall SP2(\"It's OK, It's All Good\") through the first 1000 retrieval(Q + 'Pentagram (Indian band)') results\n",
      "5ae59117554299546bf82ef1: Unable recall SP2('All the Lights in the Sky') through the first 1000 retrieval(Q + 'The Yogscast') results\n",
      "5ae626865542992663a4f26d: Unable recall SP2('Jillian Schlesinger') through the first 1000 retrieval(Q + 'Laura Dekker') results\n",
      "5a89c120554299669944a597: Unable recall SP2('Laurdine Patrick') through the first 1000 retrieval(Q + 'Deval Patrick') results\n",
      "5ab29789554299194fa93419: Unable recall SP2('Eric Clapton singles discography') through the first 1000 retrieval(Q + 'Tears in Heaven') results\n",
      "5ac495035542996feb3fe8e9: Unable recall SP2('Jelena Gavrilović') through the first 1000 retrieval(Q + 'A Serbian Film') results\n",
      "5abc8697554299700f9d78e5: Unable recall SP2('Kwabre West (Ghana parliament constituency)') through the first 1000 retrieval(Q + 'Kwabre District') results\n",
      "5a8a409c55429930ff3c0d27: Unable recall SP2('Electoral district of Ivanhoe') through the first 1000 retrieval(Q + 'Ivanhoe, Victoria') results\n",
      "5ac2c931554299657fa29059: Unable recall SP2('Philip Hill-Pearson') through the first 1000 retrieval(Q + 'United (2011 film)') results\n",
      "5a7a600155429941d65f25ee: Unable recall SP2('VAQ-130') through the first 1000 retrieval(Q + 'Naval Air Station Whidbey Island') results\n",
      "5ade274e5542997c77aded79: Unable recall SP2('So You Think You Can Dance (U.S. season 5)') through the first 1000 retrieval(Q + 'Jeanine Mason') results\n",
      "5adcdeb65542992c1e3a2442: Unable recall SP2('Annette Charles') through the first 1000 retrieval(Q + 'Grease (film)') results\n",
      "5ae23ab45542994d89d5b3a0: Unable recall SP2('The Lancashire Hotpots') through the first 1000 retrieval(Q + 'Colin Murray') results\n",
      "5a7e88ff55429949594199f7: Unable recall SP2('Mother Mother (song)') through the first 1000 retrieval(Q + 'Royals (song)') results\n",
      "5ac2544e5542992f1f2b3857: Unable recall SP2('Juanito (footballer, born 1990)') through the first 1000 retrieval(Q + 'Xerez CD') results\n",
      "5ae74ffa5542991bbc9761e3: Unable recall SP2('Serra Mesa, San Diego') through the first 1000 retrieval(Q + 'Mission San Diego de Alcalá') results\n",
      "5a7b22225542992d025e6786: Unable recall SP2('Lessness (short story)') through the first 1000 retrieval(Q + 'John McCarthy (journalist)') results\n",
      "5a85fab35542997b5ce40044: Unable recall SP2('U2 discography') through the first 1000 retrieval(Q + 'Sunday Bloody Sunday') results\n",
      "5ae252675542992decbdccb4: Unable recall SP2('The Gossip (EP)') through the first 1000 retrieval(Q + 'Gossip (band)') results\n",
      "5ae3b9365542990afbd1e1bc: Unable recall SP2('Adam I. Lapidus') through the first 1000 retrieval(Q + 'The Front (The Simpsons)') results\n",
      "5a805df45542996402f6a4f9: Unable recall SP2('St. Louis Woman') through the first 1000 retrieval(Q + 'Harold Nicholas') results\n",
      "5a89fedc55429970aeb701ec: Unable recall SP2('Definite Article') through the first 1000 retrieval(Q + 'Eddie Izzard') results\n",
      "5ae786a855429952e35ea961: Unable recall SP2('Who Is Clark Rockefeller?') through the first 1000 retrieval(Q + 'Christian Gerhartsreiter') results\n",
      "5ae409d05542996836b02c20: Unable recall SP2(\"Everybody's Talkin'\") through the first 1000 retrieval(Q + 'The Hangover Part III') results\n",
      "5a89cd46554299669944a5b2: Unable recall SP2('Orenda OE600') through the first 1000 retrieval(Q + 'Orenda Engines') results\n",
      "5ab7ddb45542995dae37e9c6: Unable recall SP2('The Country Bears') through the first 1000 retrieval(Q + 'Country Bear Jamboree') results\n",
      "5a806138554299485f5985dd: Unable recall SP2('David Wharnsby') through the first 1000 retrieval(Q + 'Sarah Polley') results\n",
      "5ac160165542991316484b01: Unable recall SP2('Paulding County, Ohio') through the first 1000 retrieval(Q + 'John André') results\n",
      "5ae4706d55429970de88d95f: Unable recall SP2('Jataka Bharanam') through the first 1000 retrieval(Q + 'Mudrarakshasa') results\n",
      "5ac3b86c554299204fd21e16: Unable recall SP2('Alan Jones (film critic)') through the first 1000 retrieval(Q + 'Premiere') results\n",
      "5ac39cb5554299391541381c: Unable recall SP2('HNoMS King Haakon VII') through the first 1000 retrieval(Q + 'Royal Norwegian Navy') results\n",
      "5ac5414e5542993e66e8226b: Unable recall SP2('Doug Howard (musician)') through the first 1000 retrieval(Q + 'He-Man') results\n",
      "5ae202ad5542994d89d5b300: Unable recall SP2('Marian Seldes') through the first 1000 retrieval(Q + 'American Theater Hall of Fame') results\n",
      "5a7e2b505542995ed0d166bd: Unable recall SP2('John William Whittaker') through the first 1000 retrieval(Q + 'Edmund Sharpe') results\n",
      "5ae0823e55429906c02daad7: Unable recall SP2('Pentagon Renovation Program') through the first 1000 retrieval(Q + 'The Pentagon') results\n",
      "5a7b80f255429927d897bfdf: Unable recall SP2('Do You Remember These') through the first 1000 retrieval(Q + 'The Statler Brothers') results\n",
      "5addf42c5542997545bbbddd: Unable recall SP2('Bungay Priory') through the first 1000 retrieval(Q + 'Dissolution of the Monasteries') results\n",
      "5a8b25c855429950cd6afc57: Unable recall SP2('Value America') through the first 1000 retrieval(Q + 'Vulcan Inc.') results\n",
      "5adc7d2b5542994d58a2f616: Unable recall SP2('Katy Perry discography') through the first 1000 retrieval(Q + 'E.T. (song)') results\n",
      "5ac14b1555429964131be1a2: Unable recall SP2('Recluse') through the first 1000 retrieval(Q + 'Greta Garbo') results\n",
      "5ae1361e55429920d5234320: Unable recall SP2('1968 Minnesota Vikings season') through the first 1000 retrieval(Q + 'Bud Grant') results\n",
      "5add2fc95542992ae4cec4dd: Unable recall SP2('World Waterpark') through the first 1000 retrieval(Q + 'Tropical Islands Resort') results\n",
      "5abae9dc5542996606241683: Unable recall SP2('Ketu South Municipal District') through the first 1000 retrieval(Q + 'Ketu North District') results\n",
      "5a811d9f554299260e20a245: Unable recall SP2('Sarah Gadon') through the first 1000 retrieval(Q + 'My Dad the Rock Star') results\n",
      "5a87f58e55429938390d3ec6: Unable recall SP2('Eric Carlson (musician)') through the first 1000 retrieval(Q + 'Eldon Hoke') results\n",
      "5ac22c4155429951e9e6848c: Unable recall SP2('John Philip Kassebaum') through the first 1000 retrieval(Q + 'Nancy Kassebaum') results\n",
      "5a8f25a45542995b44241f79: Unable recall SP2('Grass Stains') through the first 1000 retrieval(Q + 'Tye Sheridan') results\n",
      "5a812dee55429938b61422e2: Unable recall SP2(\"Waymore's Blues (Part II)\") through the first 1000 retrieval(Q + 'The Road Goes On Forever (The Highwaymen album)') results\n",
      "5abd7bf855429924427fd001: Unable recall SP2('Inspired (song)') through the first 1000 retrieval(Q + 'Younger Now') results\n",
      "5a86ab655542991e7718167d: Unable recall SP2('Purple Numbers') through the first 1000 retrieval(Q + 'Douglas Engelbart') results\n",
      "5ae58a625542993aec5ec1b4: Unable recall SP2('Finding Mr. Right') through the first 1000 retrieval(Q + 'Sleepless in Seattle') results\n",
      "5a7fe5ca5542994857a76835: Unable recall SP2('Roger Ruzek') through the first 1000 retrieval(Q + 'Chris Berman') results\n",
      "5ab2b513554299340b525549: Unable recall SP2('Mental Floss') through the first 1000 retrieval(Q + \"Dr. Dobb's Journal\") results\n",
      "5a758c305542992db947367a: Unable recall SP2('Bibury') through the first 1000 retrieval(Q + 'Mini-Europe') results\n",
      "5ab3c10f55429969a97a81ae: Unable recall SP2('Charlie Chan') through the first 1000 retrieval(Q + 'Fu Manchu') results\n",
      "5ac45947554299076e296da3: Unable recall SP2('12th Lumières Awards') through the first 1000 retrieval(Q + 'Tell No One') results\n",
      "5ab1dbde5542993be8fa985c: Unable recall SP2('Aleksandr Pishvanov') through the first 1000 retrieval(Q + 'Walt Disney') results\n",
      "5ab2853d554299449642c8db: Unable recall SP2('Cultösaurus Erectus') through the first 1000 retrieval(Q + 'Blue Öyster Cult') results\n",
      "5a7b2c1f55429927d897bf4f: Unable recall SP2('Château Cany') through the first 1000 retrieval(Q + 'Louis XIII of France') results\n",
      "5a89e60b5542992e4fca8460: Unable recall SP2('Paluke Bangaaramaayena') through the first 1000 retrieval(Q + 'Kancherla Gopanna') results\n",
      "5ab974c455429970cfb8eb07: Unable recall SP2('All Around the World (Paulina Rubio song)') through the first 1000 retrieval(Q + 'Zumba') results\n",
      "5ab795065542995dae37e96a: Unable recall SP2('Neil Doolan') through the first 1000 retrieval(Q + 'North Melbourne Football Club') results\n",
      "5a8f4ce45542995b4424200d: Unable recall SP2('Finniss Ministry') through the first 1000 retrieval(Q + 'B. T. Finniss') results\n",
      "5ae1f8bd5542994d89d5b2e6: Unable recall SP2('Chevrolet Camaro (first generation)') through the first 1000 retrieval(Q + 'V8 engine') results\n",
      "5a7febe95542994857a76851: Unable recall SP2('UTV Movies') through the first 1000 retrieval(Q + 'UTV Software Communications') results\n",
      "5ab8eefc5542991b5579f058: Unable recall SP2('Storms of My Grandchildren') through the first 1000 retrieval(Q + 'James Hansen') results\n",
      "5a7fe4765542994857a7682d: Unable recall SP2(\"Ain't Nuttin' But Music\") through the first 1000 retrieval(Q + 'D12') results\n",
      "5abe97b75542993f32c2a176: Unable recall SP2('Garrus Vakarian') through the first 1000 retrieval(Q + 'Party (role-playing games)') results\n",
      "5a8b57dc5542995d1e6f1370: Unable recall SP2('Brynne: My Bedazzled Life') through the first 1000 retrieval(Q + 'Geoffrey Edelsten') results\n",
      "5a8f20105542997ba9cb31d9: Unable recall SP2('A Cherry Cherry Christmas') through the first 1000 retrieval(Q + 'The Chanukah Song') results\n",
      "5add3d4e5542997545bbbceb: Unable recall SP2('Dwight Henry (actor)') through the first 1000 retrieval(Q + 'Beasts of the Southern Wild') results\n",
      "5a7b48eb55429931da12ca56: Unable recall SP2('Neighbourhoods of Buenos Aires') through the first 1000 retrieval(Q + 'La Boca') results\n",
      "5a76386b5542994ccc918721: Unable recall SP2('The Ricks Must Be Crazy') through the first 1000 retrieval(Q + 'The Gods Must Be Crazy') results\n",
      "5a8871b6554299206df2b26f: Unable recall SP2('Britannia Superior') through the first 1000 retrieval(Q + 'Londinium') results\n",
      "5abd8eda5542996e802b47cd: Unable recall SP2('Desperado (song)') through the first 1000 retrieval(Q + 'Desperado (Eagles album)') results\n",
      "5ac4e73555429919431739f3: Unable recall SP2('Alcubierre drive') through the first 1000 retrieval(Q + 'Miguel Alcubierre') results\n",
      "5ab9440e554299743d22ea9d: Unable recall SP2('The Best of Rascal Flatts Live') through the first 1000 retrieval(Q + 'Bless the Broken Road') results\n",
      "5a79658855429970f5fffea5: Unable recall SP2('Korean Blockbuster Movies') through the first 1000 retrieval(Q + 'Shiri (film)') results\n",
      "5ab521615542991779162d88: Unable recall SP2('Boneland') through the first 1000 retrieval(Q + 'Jodrell Bank Observatory') results\n",
      "5a7ba8a7554299294a54aa6e: Unable recall SP2('Tuborg Image Awards') through the first 1000 retrieval(Q + 'Tuborg Brewery') results\n",
      "5adf4b435542993a75d264a6: Unable recall SP2('Dream Scene (song)') through the first 1000 retrieval(Q + 'Wonderwall Music') results\n",
      "5a7f413f55429930675136e0: Unable recall SP2(\"Dumplin' (film)\") through the first 1000 retrieval(Q + 'Dove Cameron') results\n",
      "5ac2c5e7554299218029db43: Unable recall SP2('Battle Mountains, Nevada') through the first 1000 retrieval(Q + 'Battle Mountain, Nevada') results\n",
      "5ae7a62355429952e35ea9a3: Unable recall SP2('Stevens County, Kansas') through the first 1000 retrieval(Q + 'Thaddeus Stevens') results\n",
      "5ae74cee5542997b22f6a69a: Unable recall SP2('2 of One') through the first 1000 retrieval(Q + '...And Justice for All (album)') results\n",
      "5a89b3bd554299669944a572: Unable recall SP2('How to Pick Up Girls') through the first 1000 retrieval(Q + 'Desi Arnaz Jr.') results\n",
      "5adfbb5755429942ec259b23: Unable recall SP2('Iolanta') through the first 1000 retrieval(Q + 'Modest Ilyich Tchaikovsky') results\n",
      "5ab7eac55542995dae37e9f1: Unable recall SP2(\"2010 NCAA Division I Men's Basketball Tournament\") through the first 1000 retrieval(Q + 'RCA Dome') results\n",
      "5a8592155542992a431d1b26: Unable recall SP2('The Man Who Never Was') through the first 1000 retrieval(Q + 'Operation Mincemeat') results\n",
      "5a72d73d5542992359bc31aa: Unable recall SP2('Ove Pihl') through the first 1000 retrieval(Q + 'Atlantis') results\n",
      "5adfe5a655429942ec259b73: Unable recall SP2('Billy Wilson (footballer, born 1936)') through the first 1000 retrieval(Q + 'Linfield F.C.') results\n",
      "5ab6aa40554299710c8d1f34: Unable recall SP2('5.6×52mmR') through the first 1000 retrieval(Q + 'Savage Arms') results\n",
      "5ac24c8f5542996366519962: Unable recall SP2('Nathaniel Hawthorne Birthplace') through the first 1000 retrieval(Q + 'House of the Seven Gables') results\n",
      "5a7b3fca5542992d025e67be: Unable recall SP2('Breakthrough (Lemonade Mouth song)') through the first 1000 retrieval(Q + 'Naomi Scott') results\n",
      "5ac1a1275542991316484b69: Unable recall SP2('Review of the Labour Party in Scotland') through the first 1000 retrieval(Q + 'Ed Miliband') results\n",
      "5ab7ecee5542991d322237c3: Unable recall SP2('Will Smith discography') through the first 1000 retrieval(Q + \"Gettin' Jiggy wit It\") results\n",
      "5a81a4e25542990a1d231e3c: Unable recall SP2('Mobile army surgical hospital (US)') through the first 1000 retrieval(Q + 'M*A*S*H (TV series)') results\n",
      "5a8b432b55429949d91db519: Unable recall SP2('Maxwell Becton') through the first 1000 retrieval(Q + 'Becton Dickinson') results\n",
      "5a87d9205542993e715abfe6: Unable recall SP2('Quercus oglethorpensis') through the first 1000 retrieval(Q + 'James Oglethorpe') results\n",
      "5a89cdbe554299669944a5b5: Unable recall SP2('Epworth by the Sea') through the first 1000 retrieval(Q + 'Epworth, Lincolnshire') results\n",
      "5ab29f82554299194fa93479: Unable recall SP2('Grinnell Willis') through the first 1000 retrieval(Q + 'Nathaniel Parker Willis') results\n",
      "5a8b43a65542997f31a41cc9: Unable recall SP2('Hillsboro Fire Department') through the first 1000 retrieval(Q + 'Hillsboro, Oregon') results\n",
      "5ae3ade15542992f92d82339: Unable recall SP2('Billie Lourd') through the first 1000 retrieval(Q + 'Bryan Lourd') results\n",
      "5a7275fd5542994cef4bc2b0: Unable recall SP2('A Connecticut Party') through the first 1000 retrieval(Q + 'United States Senate') results\n",
      "5a90b75c5542990a984936c4: Unable recall SP2('Southwest Harbor Public Library') through the first 1000 retrieval(Q + 'Southwest Harbor, Maine') results\n",
      "5a8b5b755542997f31a41ce9: Unable recall SP2('Bajaj Kawasaki Wind 125') through the first 1000 retrieval(Q + 'Bajaj Auto') results\n",
      "5a8a5640554299515336136c: Unable recall SP2('Boston Society of Natural History') through the first 1000 retrieval(Q + 'Back Bay, Boston') results\n",
      "5ac28c8e5542996773102581: Unable recall SP2(\"L'amore si muove\") through the first 1000 retrieval(Q + 'Il Volo') results\n",
      "5ae5608a5542992663a4f1ef: Unable recall SP2('Carl Wilhelm Scheele') through the first 1000 retrieval(Q + 'Oxalic acid') results\n",
      "5ab3c57c55429969a97a81ba: Unable recall SP2('Poovarasam Peepee') through the first 1000 retrieval(Q + 'Thespesia populnea') results\n",
      "5ade4c095542997c77adedd9: Unable recall SP2('Hemi-Sync') through the first 1000 retrieval(Q + 'Robert Monroe') results\n",
      "Unable recall SP1 through the first 1000 retrieval results: 5abcb387554299700f9d792e\n",
      "5ae4ede455429960a22e020c: Unable recall SP2('Rocky Mountain cuisine') through the first 1000 retrieval(Q + 'Rocky Mountain oysters') results\n",
      "5adbf69755429947ff173870: Unable recall SP2('Jefferson Kaye') through the first 1000 retrieval(Q + 'WBZ (AM)') results\n",
      "5ae263b8554299495565da59: Unable recall SP2('Vital Tomosi Dairy Limited') through the first 1000 retrieval(Q + 'Kampala') results\n",
      "Unable recall SP1 through the first 1000 retrieval results: 5abbdd355542992ccd8e7fc6\n",
      "5abfd0505542993fe9a41e81: Unable recall SP2('Mineral-insulated copper-clad cable') through the first 1000 retrieval(Q + 'Pentair') results\n",
      "5a7fe0275542994857a76815: Unable recall SP2('Ming Xi') through the first 1000 retrieval(Q + 'Traditional Chinese characters') results\n",
      "5a8b4db255429949d91db53c: Unable recall SP2('Nick Sakai') through the first 1000 retrieval(Q + 'Betsy Brandt') results\n",
      "5a8f658b5542992414482aaf: Unable recall SP2('ARA Drummond') through the first 1000 retrieval(Q + 'Drummond-class corvette') results\n",
      "5a88bb5255429946c8d6e89e: Unable recall SP2('REMIXXXX') through the first 1000 retrieval(Q + 'XXXX (album)') results\n",
      "5ae2c29f554299495565db2b: Unable recall SP2('Roosevelt Field (shopping mall)') through the first 1000 retrieval(Q + 'Destiny USA') results\n",
      "5adc901e554299438c868dfd: Unable recall SP2('Caricia') through the first 1000 retrieval(Q + 'Leo Dan') results\n",
      "5adc5b6b55429947ff173933: Unable recall SP2('9999 Wiles') through the first 1000 retrieval(Q + 'Andrew Wiles') results\n",
      "5ab2c1e8554299545a2cfa5a: Unable recall SP2('A Modest Video Game Proposal') through the first 1000 retrieval(Q + 'A Modest Proposal') results\n",
      "5a821b59554299676cceb217: Unable recall SP2('Scentimental Romeo') through the first 1000 retrieval(Q + 'Penelope Pussycat') results\n",
      "5ab28cb55542993be8fa9944: Unable recall SP2('The Actors') through the first 1000 retrieval(Q + 'Michael Gambon') results\n",
      "5ac16fda55429964131be1f9: Unable recall SP2('Thicker Than Water (Under the Dome)') through the first 1000 retrieval(Q + 'Under the Dome (TV series)') results\n",
      "5a8f142455429918e830d182: Unable recall SP2('Shanghai, Shanghai') through the first 1000 retrieval(Q + 'Yuen Biao') results\n",
      "5a7d1a3f554299452d57baed: Unable recall SP2('Another Suitcase in Another Hall') through the first 1000 retrieval(Q + 'Eva Perón') results\n",
      "5a88f404554299206df2b3d1: Unable recall SP2('Augustine Abbey (Idikoko)') through the first 1000 retrieval(Q + 'Esi Sutherland-Addy') results\n",
      "5ae0f4fe55429945ae959491: Unable recall SP2('Bad Samaritan (film)') through the first 1000 retrieval(Q + 'Robert Sheehan') results\n",
      "5a8bbae05542995e66a474e6: Unable recall SP2('Maud (ship)') through the first 1000 retrieval(Q + 'Maud of Wales') results\n",
      "5a851d4a5542992a431d1adf: Unable recall SP2('Caustic Window (album)') through the first 1000 retrieval(Q + 'Discogs') results\n",
      "5ac1c9ae554299636651990e: Unable recall SP2('Faye Abbott') through the first 1000 retrieval(Q + 'Kenyon College') results\n",
      "5a88710b554299206df2b26b: Unable recall SP2('Last Rights (album)') through the first 1000 retrieval(Q + 'Revolution 9') results\n",
      "5a8ce0d5554299441c6b9f4c: Unable recall SP2('Macau International Television Festival') through the first 1000 retrieval(Q + 'Macau International Movie Festival') results\n",
      "5a8e25465542990e94052a74: Unable recall SP2('The Hunting Party Tour') through the first 1000 retrieval(Q + 'Of Mice & Men (band)') results\n",
      "5ac2595155429951e9e6856c: Unable recall SP2('Karle Pyaar Karle') through the first 1000 retrieval(Q + 'Love Me If You Dare') results\n",
      "5ab7b2925542995dae37e992: Unable recall SP2('History of the New York State College of Forestry') through the first 1000 retrieval(Q + 'Ithaca, New York') results\n",
      "5ae341c05542994393b9e66a: Unable recall SP2('M*A*S*H (season 8)') through the first 1000 retrieval(Q + 'M*A*S*H (TV series)') results\n",
      "5ac2280c5542992f1f2b381e: Unable recall SP2('Save the Badger Badger Badger') through the first 1000 retrieval(Q + 'Brian May') results\n",
      "5a8b22dd55429950cd6afc54: Unable recall SP2('Rocky Iaukea') through the first 1000 retrieval(Q + 'King Curtis Iaukea') results\n",
      "5a903aeb55429933b8a204ab: Unable recall SP2('Sutera') through the first 1000 retrieval(Q + 'Haemanthus') results\n",
      "5a79f84b5542996c55b2dcb9: Unable recall SP2('Bejan number') through the first 1000 retrieval(Q + 'Adrian Bejan') results\n",
      "5abd0f975542992ac4f381bc: Unable recall SP2('Marco Angeli di Sartèna') through the first 1000 retrieval(Q + 'Literary Review') results\n",
      "5a8ac02955429930ff3c0e0d: Unable recall SP2('John Koenig') through the first 1000 retrieval(Q + 'Martin Landau') results\n",
      "5ae5bd3155429960a22e0325: Unable recall SP2('James River Steam Brewery Cellars') through the first 1000 retrieval(Q + 'Yuengling') results\n",
      "5ac4396f5542995c82c4acfb: Unable recall SP2('Lara Grice') through the first 1000 retrieval(Q + 'The Final Destination') results\n",
      "5ac2b52f55429921a00ab041: Unable recall SP2('2012 Stanford Cardinal football team') through the first 1000 retrieval(Q + 'David Shaw (American football)') results\n",
      "5a79f4ee5542996c55b2dcb0: Unable recall SP2('Californication Tour') through the first 1000 retrieval(Q + 'Californication (album)') results\n",
      "5ab2afbf55429929539467fa: Unable recall SP2('Ave Maria University') through the first 1000 retrieval(Q + 'Ave Maria, Florida') results\n",
      "5abab18a55429955dce3eecf: Unable recall SP2('Pilot (Prison Break)') through the first 1000 retrieval(Q + 'Brett Ratner') results\n",
      "5ac0091d554299012d1db580: Unable recall SP2('I, Tonya') through the first 1000 retrieval(Q + 'Tonya Harding') results\n",
      "5abbcff35542993f40c73bab: Unable recall SP2('Take On the World (Judas Priest song)') through the first 1000 retrieval(Q + 'We Will Rock You') results\n",
      "5ae2c2a55542992decbdcd97: Unable recall SP2('Mantra (restaurant)') through the first 1000 retrieval(Q + 'Voss (water)') results\n",
      "5ae149b755429920d5234396: Unable recall SP2('Wake, Rattle, and Roll') through the first 1000 retrieval(Q + 'Shake, Rattle and Roll') results\n",
      "5ac44e835542995c82c4ad26: Unable recall SP2('Efraín Juárez') through the first 1000 retrieval(Q + 'C.F. Monterrey') results\n",
      "5a772ca155429972597f1462: Unable recall SP2('Live! You Goddamned Son of a Bitch') through the first 1000 retrieval(Q + 'Big Sexy Land') results\n",
      "5ac49f315542996feb3fe91c: Unable recall SP2('Haugtussa') through the first 1000 retrieval(Q + 'Hulder') results\n",
      "5a8c693f554299585d9e368b: Unable recall SP2('Clydesdale horse') through the first 1000 retrieval(Q + 'Budweiser Clydesdales') results\n",
      "5adf70bb5542995ec70e900f: Unable recall SP2('Undeniable: Evolution and the Science of Creation') through the first 1000 retrieval(Q + 'Bill Nye–Ken Ham debate') results\n",
      "5a8f57b15542992414482a67: Unable recall SP2('Gaga (plant)') through the first 1000 retrieval(Q + 'Lady Gaga') results\n",
      "5ade17f055429975fa854e37: Unable recall SP2('Ascendancy (film)') through the first 1000 retrieval(Q + 'Irish question') results\n",
      "5ae26089554299495565da4e: Unable recall SP2('Miklos Wright') through the first 1000 retrieval(Q + 'Open Range (2003 film)') results\n",
      "5a8199625542990a1d231e1d: Unable recall SP2('Bishop Thomas K. Gorman Catholic School') through the first 1000 retrieval(Q + 'Bishop Gorman High School') results\n",
      "5adc79d95542994650320d1b: Unable recall SP2('RNLB Guide of Dunkirk (ON 826)') through the first 1000 retrieval(Q + 'Little Ships of Dunkirk') results\n",
      "5ac35b935542995ef918c180: Unable recall SP2('Summer Heat (2008 film)') through the first 1000 retrieval(Q + 'Turkish Delight (1973 film)') results\n",
      "5a770fa955429937353601bc: Unable recall SP2('Kashi Kicks') through the first 1000 retrieval(Q + 'Joey Porter') results\n",
      "5ab83f3f55429919ba4e2286: Unable recall SP2('Viper (Gotham)') through the first 1000 retrieval(Q + 'Jada Pinkett Smith') results\n",
      "5a7178565542994082a3e84a: Unable recall SP2('Gurren Lagann') through the first 1000 retrieval(Q + 'The Secret of Kells') results\n",
      "5a89a94255429946c8d6e98d: Unable recall SP2('Football (film)') through the first 1000 retrieval(Q + 'Johnson (composer)') results\n",
      "5adfb63f55429942ec259b09: Unable recall SP2('The 440 Alliance') through the first 1000 retrieval(Q + 'Robert Downey Jr.') results\n",
      "5a7f38315542992e7d278c97: Unable recall SP2('I Married Marge') through the first 1000 retrieval(Q + 'Dr. Hibbert') results\n",
      "5ae4cb4755429908b632647a: Unable recall SP2('Griggstown cow') through the first 1000 retrieval(Q + 'Jersey Devil') results\n",
      "5a7f9a175542995d8a8ddeb9: Unable recall SP2('Fuchs') through the first 1000 retrieval(Q + 'Fox') results\n",
      "5a8cdbb2554299441c6b9f2d: Unable recall SP2('Tuborg Image Awards') through the first 1000 retrieval(Q + 'Tuborg Brewery') results\n",
      "5ae563da5542993aec5ec19f: Unable recall SP2('Erik Hivju') through the first 1000 retrieval(Q + 'Kristofer Hivju') results\n",
      "5adc39a755429944faac247a: Unable recall SP2('The Garden of Unearthly Delights') through the first 1000 retrieval(Q + 'The Garden of Earthly Delights') results\n",
      "Unable recall SP1 through the first 1000 retrieval results: 5ab6e31f554299710c8d1f97\n",
      "5adeaa405542995534e8c6ce: Unable recall SP2('Carl Blair') through the first 1000 retrieval(Q + 'Bob Jones University') results\n",
      "5a81b80955429926c1cdad68: Unable recall SP2('Sherry!') through the first 1000 retrieval(Q + 'James Lipton') results\n",
      "5ab6df6c55429953192ad3a2: Unable recall SP2('Nephté') through the first 1000 retrieval(Q + 'Galatia') results\n",
      "5abbd98655429931dba145a4: Unable recall SP2('1985 Rajneeshee assassination plot') through the first 1000 retrieval(Q + '1984 Rajneeshee bioterror attack') results\n",
      "5ab43b7b5542991751b4d6ff: Unable recall SP2('Buy Nothing Christmas') through the first 1000 retrieval(Q + 'Buy Nothing Day') results\n",
      "5ae331d455429928c423966c: Unable recall SP2('Radisson Blu Hotel, Bremen') through the first 1000 retrieval(Q + 'Radisson Hotels') results\n",
      "5a8a06f655429970aeb7021b: Unable recall SP2('Arthur White (actor)') through the first 1000 retrieval(Q + 'David Jason') results\n",
      "5a8c6e8d554299585d9e369b: Unable recall SP2('1968 Canadian Grand Prix') through the first 1000 retrieval(Q + 'Pedro Rodríguez (racing driver)') results\n",
      "5ac0b7c45542996f0d89cc20: Unable recall SP2('Elastic Love') through the first 1000 retrieval(Q + 'Electro (music)') results\n",
      "5a9092b355429916514e74fc: Unable recall SP2('Werner Wolff (photographer)') through the first 1000 retrieval(Q + 'Kehlsteinhaus') results\n",
      "5ac4accd5542996feb3fe931: Unable recall SP2('Ovvoru Pookalume') through the first 1000 retrieval(Q + 'Sindhu Bhairavi (raga)') results\n",
      "5ae15dba5542990adbacf772: Unable recall SP2('WWFG') through the first 1000 retrieval(Q + 'Ocean City, Maryland') results\n",
      "5a775f9555429966f1a36d17: Unable recall SP2('La Fayette (film)') through the first 1000 retrieval(Q + 'Gilbert du Motier, Marquis de Lafayette') results\n",
      "5a8a12975542992d82986e91: Unable recall SP2('Augher (Parliament of Ireland constituency)') through the first 1000 retrieval(Q + 'Acts of Union 1800') results\n",
      "5ab7c09f55429928e1fe38c5: Unable recall SP2('Spanglish (film)') through the first 1000 retrieval(Q + 'James L. Brooks') results\n",
      "5ae7843455429952e35ea95d: Unable recall SP2('Leni Junker') through the first 1000 retrieval(Q + \"Women's 100 metres world record progression\") results\n",
      "5a8ccb97554299441c6b9f0c: Unable recall SP2('One Night in Miami') through the first 1000 retrieval(Q + 'Sonny Liston') results\n",
      "5a8af32b55429971feec45b0: Unable recall SP2('Lake Bell') through the first 1000 retrieval(Q + 'Million Dollar Arm') results\n",
      "5a7a12c95542996c55b2dcfb: Unable recall SP2('Prince Faisal bin Hussein') through the first 1000 retrieval(Q + 'Abdullah II of Jordan') results\n",
      "5ae2efab55429928c4239579: Unable recall SP2(\"I Wouldn't Be a Man\") through the first 1000 retrieval(Q + 'Billy Dean') results\n",
      "5a7d778d5542990b8f50399c: Unable recall SP2('Lost Verizon') through the first 1000 retrieval(Q + 'When You Dish Upon a Star') results\n",
      "Unable recall SP1 through the first 1000 retrieval results: 5a72ff365542991f29ee2cf1\n",
      "5ae6296355429929b0807ae7: Unable recall SP2('Tribal Thunder') through the first 1000 retrieval(Q + 'Dick Dale') results\n",
      "5ab211d8554299340b525484: Unable recall SP2('2015 in video gaming') through the first 1000 retrieval(Q + 'Fallout 4') results\n",
      "5a90b96155429933b8a205c1: Unable recall SP2(\"Odin's People\") through the first 1000 retrieval(Q + 'Derek Thompson (actor)') results\n",
      "5ae60bd45542996de7b71b0e: Unable recall SP2('Alden Richards (EP)') through the first 1000 retrieval(Q + 'TriNoma') results\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from html import unescape\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "from utils.data_utils import get_valid_links\n",
    "\n",
    "faiss.omp_set_num_threads(1)\n",
    "\n",
    "is_strict = True\n",
    "step_data_file = open(f\"data/hotpot-step-{split}{'.strict' if is_strict else ''}.jsonl\", 'w')\n",
    "\n",
    "OBS_SIZE = 1\n",
    "RET_SIZE = 1000\n",
    "# HOP_MAX_STEPS = (RET_SIZE + OBS_SIZE - 1) // OBS_SIZE\n",
    "hotpot_filter = {\"term\": {\"for_hotpot\": True}}\n",
    "examples = []\n",
    "for q_idx, (q_id, qas) in enumerate(tqdm(samples)):\n",
    "    question, answer, sp_facts = qas\n",
    "    if len(sp_facts) < 2:\n",
    "        print(f\"less than 2 supporting facts: {q_id}\")\n",
    "    norm_sp_titles = set(unescape(t) for t in sp_facts.keys())\n",
    "    sp_ids = list(title2id[t] for t in norm_sp_titles)\n",
    "\n",
    "    hard_negs = set()\n",
    "    state2action = dict()\n",
    "\n",
    "    # ==================== initial ====================\n",
    "    sp_ranks = {strategy: {sp_id: 2 * RET_SIZE for sp_id in sp_ids}\n",
    "                for strategy in [\"BM25\", \"BM25+Link\", \"MDR\", \"MDR+Link\"]}\n",
    "\n",
    "    # BM25\n",
    "    if not query_redis.exists(question):\n",
    "        query_redis.set(question, qg1.predict(question, question)[0][0].strip())\n",
    "    q1 = query_redis.get(question)\n",
    "    if not bm25_redis.exists(q1) or (bm25_redis.llen(q1) < RET_SIZE and bm25_redis.lindex(q1, -1) != 'EOL'):\n",
    "        hits = [hit['_id'] for hit in sparse_retriever.search(q1, RET_SIZE, filter_dic=hotpot_filter, n_retrieval=RET_SIZE * 2)]\n",
    "        bm25_redis.delete(q1)\n",
    "        if len(hits) < RET_SIZE:\n",
    "            bm25_redis.rpush(q1, *(hits + ['EOL']))\n",
    "        else:\n",
    "            bm25_redis.rpush(q1, *hits)\n",
    "    bm25_hits = bm25_redis.lrange(q1, 0, -1)\n",
    "    if bm25_hits[-1] == 'EOL':\n",
    "        bm25_hits = bm25_hits[:-1]\n",
    "    for p_idx, p_id in enumerate(bm25_hits[:RET_SIZE]):\n",
    "        hyperlinks = get_valid_links(corpus[p_id], is_strict, title2id)\n",
    "        if p_id in sp_ids:\n",
    "            sp_ranks['BM25'][p_id] = min(p_idx, sp_ranks['BM25'][p_id])\n",
    "            for hard_id in bm25_hits[max(0, p_idx - 5):p_idx + 6]:\n",
    "                hard_negs.add(hard_id)\n",
    "        elif len(norm_sp_titles & set(hyperlinks.keys())) > 0:  # and p_idx < min(sp_ranks['BM25'].values()):  # xxx\n",
    "            hard_negs.add(p_id)\n",
    "        for sp_title in norm_sp_titles & set(hyperlinks.keys()):\n",
    "            sp_id = title2id[sp_title]\n",
    "            sp_ranks['BM25+Link'][sp_id] = min(p_idx, sp_ranks['BM25+Link'][sp_id])\n",
    "        if max(list(sp_ranks['BM25'].values()) + list(sp_ranks['BM25+Link'].values())) <= p_idx:\n",
    "            break\n",
    "    for hard_id in bm25_hits[:10]:\n",
    "        hard_negs.add(hard_id)\n",
    "\n",
    "    # MDR\n",
    "    qk = questions[q_idx]\n",
    "#     if not mdr_redis.exists(qk) or (mdr_redis.llen(qk) < RET_SIZE and mdr_redis.lindex(qk, -1) != 'EOL'):\n",
    "#         hits = dense_retriever.search(questions[q_idx], max(RET_SIZE, 1000), args.max_q_len)[0]\n",
    "#         mdr_redis.delete(qk)\n",
    "#         if len(hits) < RET_SIZE:\n",
    "#             mdr_redis.rpush(qk, *(hits + ['EOL']))\n",
    "#         else:\n",
    "#             mdr_redis.rpush(qk, *hits)\n",
    "    mdr_hits = mdr_redis.lrange(qk, 0, -1)\n",
    "    if mdr_hits[-1] == 'EOL':\n",
    "        mdr_hits = mdr_hits[:-1]\n",
    "    assert len(mdr_hits) > 0\n",
    "    for p_idx, p_id in enumerate(mdr_hits[:RET_SIZE]):\n",
    "        hyperlinks = get_valid_links(corpus[p_id], is_strict, title2id)\n",
    "        if p_id in sp_ids:\n",
    "            sp_ranks['MDR'][p_id] = min(p_idx, sp_ranks['MDR'][p_id])\n",
    "            for hard_id in mdr_hits[max(0, p_idx - 5):p_idx + 6]:\n",
    "                hard_negs.add(hard_id)\n",
    "        elif len(norm_sp_titles & set(hyperlinks.keys())) > 0:  # and p_idx < min(sp_ranks['MDR'].values()):  # xxx\n",
    "            hard_negs.add(p_id)\n",
    "        for sp_title in norm_sp_titles & set(hyperlinks.keys()):\n",
    "            sp_id = title2id[sp_title]\n",
    "            sp_ranks['MDR+Link'][sp_id] = min(p_idx, sp_ranks['MDR+Link'][sp_id])\n",
    "        if max(list(sp_ranks['MDR'].values()) + list(sp_ranks['MDR+Link'].values())) <= p_idx:\n",
    "            break\n",
    "    for hard_id in mdr_hits[:10]:\n",
    "        hard_negs.add(hard_id)\n",
    "\n",
    "    # get the greedy initial action\n",
    "    if min(min(_sp_ranks.values()) for _sp_ranks in sp_ranks.values()) >= RET_SIZE:\n",
    "        print(f\"Unable recall SP1 through the first {RET_SIZE} retrieval results: {q_id}\")\n",
    "        state2action['initial'] = {\"query\": q1, \"action\": \"ANSWER\", \"sp_ranks\": sp_ranks}\n",
    "    else:\n",
    "        # calculate the number of step to get SPs\n",
    "        easy_steps, hard_steps = {}, {}\n",
    "        for strategy, _sp_ranks in sp_ranks.items():\n",
    "            (easy_sp_id, easy_sp_rank), (hard_sp_id, hard_sp_rank) = sorted(_sp_ranks.items(), key=lambda x: x[1])\n",
    "            easy_steps[strategy] = (easy_sp_rank + OBS_SIZE) // OBS_SIZE\n",
    "            hard_steps[strategy] = (hard_sp_rank + OBS_SIZE) // OBS_SIZE\n",
    "            if strategy.endswith('+Link'):\n",
    "                easy_steps[strategy] += 1\n",
    "                hard_steps[strategy] += 1\n",
    "        # find the fastest strategy\n",
    "        #best_strategy = min(easy_steps.keys(), key=lambda k: (easy_steps[k], hard_steps[k]))\n",
    "        #state2action['initial'] = {\"query\": q1, \"action\": \"BM25\" if best_strategy.startswith('BM25') else \"MDR\", \"sp_ranks\": sp_ranks}\n",
    "        best_ret_func = min(['BM25', 'MDR'], key=lambda k: sorted(list(sp_ranks[k].values()) + [x + 1 for x in sp_ranks[f'{k}+Link'].values()]))\n",
    "        state2action['initial'] = {\"query\": q1, \"action\": best_ret_func, \"sp_ranks\": sp_ranks}\n",
    "\n",
    "    # ==================== partial ====================\n",
    "    for (sp1_id, sp2_id) in [sp_ids, sp_ids[::-1]]:\n",
    "        sp1 = corpus[sp1_id]\n",
    "        norm_sp1_title = unescape(sp1['title'])\n",
    "        sp2 = corpus[sp2_id]\n",
    "        norm_sp2_title = unescape(sp2['title'])\n",
    "        sp2_ranks = {strategy: 2 * RET_SIZE for strategy in [\"BM25\", \"BM25+Link\", \"MDR\", \"MDR+Link\"]}\n",
    "\n",
    "        # BM25\n",
    "        obs = ' '.join([question, f\"<t> {sp1['title']} </t> {sp1['text'][sp1['sentence_spans'][0][0]:sp1['sentence_spans'][-1][1]]}\"])\n",
    "        if not query_redis.exists(obs):\n",
    "            query_redis.set(obs, qg2.predict(obs, question)[0][0].strip())\n",
    "        q2 = query_redis.get(obs)\n",
    "        if not bm25_redis.exists(q2) or (bm25_redis.llen(q2) < RET_SIZE and bm25_redis.lindex(q2, -1) != 'EOL'):\n",
    "            hits = [hit['_id'] for hit in sparse_retriever.search(q2, RET_SIZE, filter_dic=hotpot_filter, n_retrieval=RET_SIZE * 2)]\n",
    "            bm25_redis.delete(q2)\n",
    "            if len(hits) < RET_SIZE:\n",
    "                bm25_redis.rpush(q2, *(hits + ['EOL']))\n",
    "            else:\n",
    "                bm25_redis.rpush(q2, *hits)\n",
    "        bm25_hits = bm25_redis.lrange(q2, 0, -1)\n",
    "        if bm25_hits[-1] == 'EOL':\n",
    "            bm25_hits = bm25_hits[:-1]\n",
    "        for p_idx, p_id in enumerate(bm25_hits[:RET_SIZE]):\n",
    "            hyperlinks = get_valid_links(corpus[p_id], is_strict, title2id)\n",
    "            if p_id == sp2_id:\n",
    "                sp2_ranks['BM25'] = min(p_idx, sp2_ranks['BM25'])\n",
    "                for hard_id in bm25_hits[max(0, p_idx - 5):p_idx + 6]:\n",
    "                    hard_negs.add(hard_id)\n",
    "            elif norm_sp2_title in hyperlinks.keys():  # and p_idx < sp2_ranks['BM25']:\n",
    "                hard_negs.add(p_id)\n",
    "                sp2_ranks['BM25+Link'] = min(p_idx, sp2_ranks['BM25+Link'])\n",
    "            if max(sp2_ranks['BM25'], sp2_ranks['BM25+Link']) <= p_idx:\n",
    "                break\n",
    "        for hard_id in bm25_hits[:10]:\n",
    "            hard_negs.add(hard_id)\n",
    "\n",
    "        # MDR\n",
    "        qk = f\"{questions[q_idx]}\\t+++\\t{unescape(sp1['title'])}\"\n",
    "        if not mdr_redis.exists(qk) or (mdr_redis.llen(qk) < RET_SIZE and mdr_redis.lindex(qk, -1) != 'EOL'):\n",
    "            expansion = sp1['text']\n",
    "            expansion = expansion[sp1['sentence_spans'][0][0]:sp1['sentence_spans'][-1][1]]  # if strict\n",
    "            expanded_query = (questions[q_idx], expansion if expansion else sp1['title'])\n",
    "            hits = dense_retriever.search(expanded_query, max(RET_SIZE, 1000), args.max_q_sp_len)[0]\n",
    "            mdr_redis.delete(qk)\n",
    "            if len(hits) < RET_SIZE:\n",
    "                mdr_redis.rpush(qk, *(hits + ['EOL']))\n",
    "            else:\n",
    "                mdr_redis.rpush(qk, *hits)\n",
    "        mdr_hits = mdr_redis.lrange(qk, 0, -1)\n",
    "        if mdr_hits[-1] == 'EOL':\n",
    "            mdr_hits = mdr_hits[:-1]\n",
    "        assert len(mdr_hits) > 0\n",
    "        for p_idx, p_id in enumerate(mdr_hits[:RET_SIZE]):\n",
    "            hyperlinks = get_valid_links(corpus[p_id], is_strict, title2id)\n",
    "            if p_id == sp2_id:\n",
    "                sp2_ranks['MDR'] = min(p_idx, sp2_ranks['MDR'])\n",
    "                for hard_id in mdr_hits[max(0, p_idx - 5):p_idx + 6]:\n",
    "                    hard_negs.add(hard_id)\n",
    "            elif norm_sp2_title in hyperlinks.keys():  # p_idx < sp2_ranks['MDR'] and \n",
    "                hard_negs.add(p_id)\n",
    "                sp2_ranks['MDR+Link'] = min(p_idx, sp2_ranks['MDR+Link'])\n",
    "            if max(sp2_ranks['MDR'], sp2_ranks['MDR+Link']) <= p_idx:\n",
    "                break\n",
    "        for hard_id in mdr_hits[:10]:\n",
    "            hard_negs.add(hard_id)\n",
    "\n",
    "        # get the greedy subsequent action\n",
    "        if norm_sp2_title in set(unescape(t) for t in get_valid_links(sp1, is_strict, title2id).keys()):\n",
    "            state2action[norm_sp1_title] = {\"query\": q2, \"action\": \"LINK\", \"sp2_ranks\": sp2_ranks}\n",
    "        else:\n",
    "            if min(sp2_ranks.values()) >= RET_SIZE:\n",
    "                print(f\"{q_id}: Unable recall SP2({repr(norm_sp2_title)}) through the first {RET_SIZE} retrieval(Q + {repr(norm_sp1_title)}) results\")\n",
    "                state2action[norm_sp1_title] = {\"query\": q2, \"action\": \"ANSWER\", \"sp2_ranks\": sp2_ranks}\n",
    "            else:\n",
    "                # calculate the number of step to get SP2\n",
    "                sp2_steps = dict()\n",
    "                for strategy, sp2_rank in sp2_ranks.items():\n",
    "                    sp2_steps[strategy] = (sp2_rank + OBS_SIZE) // OBS_SIZE\n",
    "                    if strategy.endswith('+Link'):\n",
    "                        sp2_steps[strategy] += 1\n",
    "                #best_strategy = min(sp2_steps.keys(), key=lambda k: sp2_steps[k])\n",
    "                #state2action[norm_sp1_title] = {\"query\": q2, \"action\": \"BM25\" if best_strategy.startswith('BM25') else \"MDR\", \"sp2_ranks\": sp2_ranks}\n",
    "                best_ret_func = min(['BM25', 'MDR'], key=lambda k: sorted([sp2_steps[k], sp2_steps[f'{k}+Link']]))\n",
    "                state2action[norm_sp1_title] = {\"query\": q2, \"action\": best_ret_func, \"sp2_ranks\": sp2_ranks}\n",
    "    \n",
    "    # add SPs' out-neighbors to hard negatives\n",
    "    for sp_id in sp_ids:\n",
    "        for out_title in get_valid_links(corpus[sp_id], is_strict, title2id):\n",
    "            hard_negs.add(title2id[out_title])\n",
    "    # remove SPs from hard negatives\n",
    "    hard_negs = hard_negs - set(sp_ids)\n",
    "    \n",
    "    example = {\n",
    "        \"_id\": q_id,\n",
    "        \"question\": question,\n",
    "        \"answer\": answer,\n",
    "        \"sp_facts\": {unescape(t): sents for t, sents in sp_facts.items()},\n",
    "        \"hard_negs\": list(hard_negs),  # in- and out- neighbors of SPs, top ranked passages\n",
    "        \"state2action\": state2action\n",
    "    }\n",
    "    examples.append(example)\n",
    "    step_data_file.write(json.dumps(example, ensure_ascii=False) + '\\n')\n",
    "\n",
    "step_data_file.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  },
  "toc-autonumbering": false,
  "toc-showcode": false,
  "toc-showtags": false,
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}