Text classification with a CNN in PyTorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text classification with a CNN in PyTorch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_PATH = \"data/text_classification/20newsgroups_train.tsv\"\n",
    "DEV_PATH = \"data/text_classification/20newsgroups_dev.tsv\"\n",
    "TEST_PATH = \"data/text_classification/20newsgroups_test.tsv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_20newsgroups\n",
    "\n",
    "train = fetch_20newsgroups(subset=\"train\")\n",
    "target_names = train.target_names\n",
    "label2idx = {label: idx for idx, label in enumerate(target_names)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make sure you have the English spacy model installed, as this will be used for tokenization:\n",
    "    \n",
    "```\n",
    "> pip install spacy\n",
    "> python -m spacy download en\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import sys\n",
    "from torchtext.data import TabularDataset, Field, BucketIterator\n",
    "\n",
    "csv.field_size_limit(sys.maxsize)\n",
    "\n",
    "text = Field(sequential=True, tokenize=\"spacy\")\n",
    "label = Field(sequential=False, use_vocab=False, preprocessing=lambda x: label2idx[x])\n",
    "\n",
    "train_data = TabularDataset(path=TRAIN_PATH, format='tsv', fields=[('label', label), ('text', text)])\n",
    "dev_data = TabularDataset(path=DEV_PATH, format='tsv', fields=[('label', label), ('text', text)])\n",
    "test_data = TabularDataset(path=TEST_PATH, format='tsv', fields=[('label', label), ('text', text)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "VOCAB_SIZE = 30000\n",
    "\n",
    "text.build_vocab(train_data, max_size=VOCAB_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 16\n",
    "train_iter = BucketIterator(dataset=train_data, batch_size=BATCH_SIZE, shuffle=True)\n",
    "dev_iter = BucketIterator(dataset=dev_data, batch_size=BATCH_SIZE)\n",
    "test_iter = BucketIterator(dataset=test_data, batch_size=BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "\n",
    "class CNNClassifier(nn.Module):\n",
    "\n",
    "    def __init__(self, embedding_dim, filter_sizes, num_filters, vocab_size, output_size):\n",
    "        super(CNNClassifier, self).__init__()\n",
    "        \n",
    "        # 1. Embedding Layer\n",
    "        self.embeddings = nn.Embedding(vocab_size, embedding_dim)\n",
    "        \n",
    "        # 2. LSTM Layer\n",
    "        self.cnn = nn.ModuleList([nn.Conv1d(1, num_filters, (fs, embedding_dim)) for fs in filter_sizes])\n",
    "\n",
    "        # 3. Dense Layer\n",
    "        self.hidden2out = nn.Linear(num_filters*len(filter_sizes), output_size)\n",
    "        \n",
    "        # Optional dropout layer\n",
    "        self.dropout_layer = nn.Dropout(p=0.4)\n",
    "\n",
    "    def forward(self, batch_text):\n",
    "\n",
    "        embeddings = self.embeddings(batch_text)\n",
    "\n",
    "        embeddings = embeddings.transpose(0,1)  # (batch, length, embed_dim)\n",
    "        embeddings = embeddings.unsqueeze(1)    # (batch, channels, length, embed_dim)\n",
    "        conv_out = [conv(embeddings) for conv in self.cnn]  # (batch, num_filters, output_length, 1)\n",
    "        conv_out = [F.relu(t).squeeze(3) for t in conv_out]\n",
    "        conv_out = [F.max_pool1d(t, t.size(2)).squeeze(2) for t in conv_out]\n",
    "        conv_out = torch.cat(conv_out, 1)\n",
    "\n",
    "        conv_out = self.dropout_layer(conv_out)\n",
    "        final_output = self.hidden2out(conv_out)\n",
    "        return final_output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.optim as optim\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "def train(model, train_iter, dev_iter, batch_size, num_batches):\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    optimizer = optim.Adam(model.parameters())\n",
    "\n",
    "    max_epochs = 20\n",
    "    loss_history = []\n",
    "    patience = 2\n",
    "    best_state_path = None\n",
    "    for epoch in range(max_epochs):\n",
    "\n",
    "        total_loss = 0\n",
    "        predictions, correct = [], []\n",
    "        for batch in tqdm(train_iter, total=num_batches):\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            pred = model(batch.text.to(device))\n",
    "            loss = criterion(pred, batch.label.to(device))\n",
    "            total_loss += loss.item()\n",
    "\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            _, pred_indices = torch.max(pred, 1)\n",
    "            predictions += list(pred_indices.cpu().numpy())\n",
    "            correct += list(batch.label.cpu().numpy())\n",
    "\n",
    "        print(\"=== Epoch\", epoch, \"===\")\n",
    "        print(\"Total training loss:\", total_loss)\n",
    "        print(\"Training performance:\", precision_recall_fscore_support(correct, predictions, average=\"micro\"))\n",
    "        \n",
    "        total_loss = 0\n",
    "        predictions, correct = [], []\n",
    "        for batch in dev_iter:\n",
    "\n",
    "            pred = model(batch.text.to(device))\n",
    "            loss = criterion(pred, batch.label.to(device))\n",
    "            total_loss += loss.item()\n",
    "\n",
    "            _, pred_indices = torch.max(pred, 1)\n",
    "            pred_indices = list(pred_indices.cpu().numpy())\n",
    "            predictions += pred_indices\n",
    "            correct += list(batch.label.cpu().numpy())\n",
    "\n",
    "        print(\"Total development loss:\", total_loss)\n",
    "        dev_stats = precision_recall_fscore_support(correct, predictions, average=\"micro\")\n",
    "        print(\"Development performance:\", dev_stats)\n",
    "        \n",
    "        if len(loss_history) == 0 or total_loss < min(loss_history): \n",
    "            fscore = dev_stats[2]\n",
    "            path = f\"model_state_{epoch}_{round(total_loss,2)}_{round(fscore,2)}\"\n",
    "            torch.save(model.state_dict(), path)\n",
    "            best_state_path = path\n",
    "            \n",
    "        if len(loss_history) > 0 and total_loss > max(loss_history[-patience:]):\n",
    "            print(\"No improvement on development set. Finishing training.\")\n",
    "            break\n",
    "            \n",
    "        loss_history.append(total_loss)\n",
    "        \n",
    "    return best_state_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1eef0c3616f040789a11cd3214245629",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=707), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=== Epoch 0 ===\n",
      "Total training loss: 1744.2471042275429\n",
      "Training performance: (0.31474279653526605, 0.31474279653526605, 0.31474279653526605, None)\n",
      "Total development loss: 1002.7418279647827\n",
      "Development performance: (0.3990971853425385, 0.3990971853425385, 0.3990971853425385, None)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "460b0f6249114c269dc12deb74b671bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=707), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=== Epoch 1 ===\n",
      "Total training loss: 807.0345216393471\n",
      "Training performance: (0.6590065405692063, 0.6590065405692063, 0.6590065405692063, None)\n",
      "Total development loss: 741.3295960128307\n",
      "Development performance: (0.5536378120021243, 0.5536378120021243, 0.5536378120021243, None)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb66b9927ac6427f8877ea16c11d5b41",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=707), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=== Epoch 2 ===\n",
      "Total training loss: 404.7720814496279\n",
      "Training performance: (0.8332154852395263, 0.8332154852395263, 0.8332154852395263, None)\n",
      "Total development loss: 623.7967542409897\n",
      "Development performance: (0.6317047265002655, 0.6317047265002655, 0.6317047265002655, None)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f6bd22f555a483db80730f8799c9960",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=707), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=== Epoch 3 ===\n",
      "Total training loss: 214.2018345296383\n",
      "Training performance: (0.9164751635142302, 0.9164751635142302, 0.9164751635142302, None)\n",
      "Total development loss: 648.8073130249977\n",
      "Development performance: (0.6509559214020181, 0.6509559214020181, 0.6509559214020181, None)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4884f700869f4a88a0c4ae765ed0f21f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=707), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=== Epoch 4 ===\n",
      "Total training loss: 137.84940619766712\n",
      "Training performance: (0.9481173767014318, 0.9481173767014318, 0.9481173767014319, None)\n",
      "Total development loss: 665.8534139245749\n",
      "Development performance: (0.658656399362719, 0.658656399362719, 0.658656399362719, None)\n",
      "No improvement on development set. Finishing training.\n"
     ]
    }
   ],
   "source": [
    "EMBEDDING_DIM = 300\n",
    "NUM_FILTERS = 128\n",
    "FILTER_SIZES = [3,4,5]\n",
    "NUM_CLASSES = len(label2idx)\n",
    "num_batches = int(len(train_data) / BATCH_SIZE)\n",
    "\n",
    "classifier = CNNClassifier(EMBEDDING_DIM, FILTER_SIZES, NUM_FILTERS, VOCAB_SIZE+2, NUM_CLASSES)  \n",
    "\n",
    "best_state_path = train(classifier.to(device), train_iter, dev_iter, BATCH_SIZE, num_batches)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "def test(model, state_path, test_iter, batch_size, num_batches, target_names):\n",
    "    \n",
    "    model.load_state_dict(torch.load(state_path))\n",
    "    \n",
    "    predictions, correct = [], []\n",
    "    for batch in test_iter:\n",
    "\n",
    "        pred = model(batch.text.to(device))\n",
    "        _, pred_indices = torch.max(pred, 1)\n",
    "\n",
    "        pred_indices = list(pred_indices.cpu().numpy())\n",
    "        predictions += pred_indices\n",
    "        correct += list(batch.label.cpu().numpy())\n",
    "\n",
    "    print(classification_report(correct, predictions, target_names=target_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0, 20524,  ...,    24,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,   783,  ...,     1,     1,     1],\n",
      "        [    1,     1,  5396,  ...,     1,     1,     1],\n",
      "        [    1,     1, 12357,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,  5119,     0,  ..., 26543,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [   0,    0,    0,  ...,    0, 1071, 7059],\n",
      "        ...,\n",
      "        [   1,    1,  458,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,  5289, 20263,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,    24,  ...,     1,     1,     1],\n",
      "        [    1,     1,     8,  ...,     1,     1,     1],\n",
      "        [    1,     1,     0,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [   0,    0,    0,  ..., 6445,    0,    0],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,    1,    0,    1],\n",
      "        [   1,    1,    1,  ...,    1, 4420,    1],\n",
      "        [   1,    1,    1,  ...,    1,   14,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 26933,  5748,  ..., 21768, 29615,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,  9544,     1],\n",
      "        [    1,     1,     1,  ...,     1, 12819,     1],\n",
      "        [    1,     1,     1,  ...,     1, 19381,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [18887,     0,     0,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [   0, 1752,    0,  ..., 3720,    0,    0],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,     0,  ...,     0, 15713,  8245],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 20060,     0,  ...,     0, 12568,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [12139,     0,     0,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[49, 49, 49,  ..., 49, 49, 49],\n",
      "        [ 7,  7,  7,  ...,  7,  7,  7],\n",
      "        [ 0,  0,  0,  ...,  0,  0,  0],\n",
      "        ...,\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1],\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1],\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1]])\n",
      "tensor([[49, 49, 49,  ..., 49, 50, 49],\n",
      "        [ 7,  7,  7,  ...,  7,  7,  7],\n",
      "        [ 0,  0,  0,  ...,  0, 72,  0],\n",
      "        ...,\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1],\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1],\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [26022,     0,     0,  ...,  9095,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   49,  175,  ...,   49,   49,   50],\n",
      "        [   7,    7,   11,  ...,    7,    7,    7],\n",
      "        [   0,    0,  112,  ...,    0, 2757,   72],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 20496,     0,  ...,     0,     0,  1318],\n",
      "        ...,\n",
      "        [    1,     0,     1,  ...,     1,     1,     1],\n",
      "        [    1,     3,     1,  ...,     1,     1,     1],\n",
      "        [    1, 10494,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   49,   53],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [   0,   37,    0,  ...,    0,    0, 2307],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,     0,  ...,  3178,     0,     0],\n",
      "        ...,\n",
      "        [    1, 10394,     1,  ...,     1,     1,     1],\n",
      "        [    1,    21,     1,  ...,     1,     1,     1],\n",
      "        [    1, 22811,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0, 14330,  ...,     0,     0,  7133],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,     0,  ..., 19173,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [12504, 18813,  6417,  ...,     0,     0, 19024],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[49, 49, 49,  ..., 49, 49, 49],\n",
      "        [ 7,  7,  7,  ...,  7,  7,  7],\n",
      "        [ 0,  0,  0,  ...,  0,  0,  0],\n",
      "        ...,\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1],\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1],\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1]])\n",
      "tensor([[49, 49, 49,  ..., 49, 49, 49],\n",
      "        [ 7,  7,  7,  ...,  7,  7,  7],\n",
      "        [ 0,  0,  0,  ...,  0,  0,  0],\n",
      "        ...,\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1],\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1],\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [12960,     0,     0,  ...,     0, 16712,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,  8471,  ...,     0,     0, 12296],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [21908,  1595,     0,  ...,     0,  4033,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [20564,     0,     0,  ..., 17754,  6883,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [13576,     0,     0,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [10903,     0, 26405,  ...,     0,     0, 15071],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,     0,  ...,     0,     0, 16474],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     0],\n",
      "        [    1,     1,     1,  ...,     1,     1,     7],\n",
      "        [    1,     1,     1,  ...,     1,     1,     0]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [29377,  3178,  5578,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,    56,  ...,     1,     1,     1],\n",
      "        [    1,     1, 18161,  ...,     1,     1,     1],\n",
      "        [    1,     1, 27638,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,     0,  ..., 12490,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   50,   49,  ...,   49,    8,   49],\n",
      "        [   7,    7,    7,  ...,    7,    0,    7],\n",
      "        [   0,   72,    0,  ...,    0,   97,    0],\n",
      "        ...,\n",
      "        [   4,    1,    1,  ...,    1,    1,    1],\n",
      "        [  16,    1,    1,  ...,    1,    1,    1],\n",
      "        [1143,    1,    1,  ...,    1,    1,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 29645,     0,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,  7970,     1,  ...,     1,     1,     1],\n",
      "        [    1,    14,     1,  ...,     1,     1,     1],\n",
      "        [    1,  4002,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,     0,  ...,     0, 17880,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 12490,  2796,  ...,     0,     0, 21005],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0, 12927,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,  6944,  ...,     1,     1,     1],\n",
      "        [    1,     1,  2160,  ...,     1,     1,     1],\n",
      "        [    1,     1,    24,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0, 15624,  ...,     0,  9680,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    50,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [29899,    72,     0,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,    14,  ...,     1,     1,     1],\n",
      "        [    1,     1, 19304,  ...,     1,     1,     1],\n",
      "        [    1,     1,     0,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 14891,     0,  ...,     0, 15488,  5578],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[ 49,  49,  49,  ...,  49,  49,  49],\n",
      "        [  7,   7,   7,  ...,   7,   7,   7],\n",
      "        [511,   0,   0,  ...,   0,   0,   0],\n",
      "        ...,\n",
      "        [  1,   1,   1,  ...,   1,   1,   1],\n",
      "        [  1,   1,   1,  ...,   1,   1,   1],\n",
      "        [  1,   1,   1,  ...,   1,   1,   1]])\n",
      "tensor([[210,  49,  49,  ...,  49,  49,  49],\n",
      "        [ 11,   7,   7,  ...,   7,   7,   7],\n",
      "        [143,   0,   0,  ...,   0,   0,   0],\n",
      "        ...,\n",
      "        [  1,   1,   1,  ...,   1,   1,   1],\n",
      "        [  1,   1,   1,  ...,   1,   1,   1],\n",
      "        [  1,   1,   1,  ...,   1,   1,   1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [4890,    0,    0,  ...,    0,    0,    0],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,    1,  483,    1],\n",
      "        [   1,    1,    1,  ...,    1,    7,    1],\n",
      "        [   1,    1,    1,  ...,    1,    0,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [22188, 11270, 26109,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0, 26849,  ...,     0, 16540, 26636],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [21860, 22432,     0,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,   103,     1,  ...,     1,     1,     1],\n",
      "        [    1,    10,     1,  ...,     1,     1,     1],\n",
      "        [    1,  5263,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   53,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [3720,    0,    0,  ...,    0,  722, 3720],\n",
      "        ...,\n",
      "        [   4,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [22147,     0,     0,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 22788,     0,  ..., 24031,     0,  2502],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,    11],\n",
      "        [    1,     1,     1,  ...,     1,     1,  2908],\n",
      "        [    1,     1,     1,  ...,     1,     1,    14]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [ 9146, 20559,     0,  ...,    24,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 12568, 14164,  ..., 18711,  7634,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0, 12490,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [ 3720,     0,     0,  ..., 24513,     0,     0],\n",
      "        ...,\n",
      "        [    1,     3,     1,  ...,     1,     1,     1],\n",
      "        [    1,     2,     1,  ...,     1,     1,     1],\n",
      "        [    1,     0,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [   0,    0, 5289,  ...,    0,    0,    0],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1]])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [21768, 16683,     0,  ...,     0,     0,   413],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    50,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,     0,  ...,   254, 24321,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [   0,    0,    0,  ...,    0,    0,    0],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,    1,  255,    1],\n",
      "        [   1,    1,    1,  ...,    1,   27,    1],\n",
      "        [   1,    1,    1,  ...,    1, 6908,    1]])\n",
      "tensor([[49, 49, 49,  ..., 49, 49, 49],\n",
      "        [ 7,  7,  7,  ...,  7,  7,  7],\n",
      "        [ 0,  0,  0,  ...,  0,  0,  0],\n",
      "        ...,\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1],\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1],\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1]])\n",
      "tensor([[ 50,  49,  49,  ...,  49,  50,  49],\n",
      "        [  7,   7,   7,  ...,   7,   7,   7],\n",
      "        [ 72,   0,   0,  ...,   0,  72,   0],\n",
      "        ...,\n",
      "        [  1,   1,   1,  ..., 591,   1,   1],\n",
      "        [  1,   1,   1,  ...,   2,   1,   1],\n",
      "        [  1,   1,   1,  ...,   0,   1,   1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,  5578,  ..., 16683,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,    56,  ...,     1,     1,     1],\n",
      "        [    1,     1, 18161,  ...,     1,     1,     1],\n",
      "        [    1,     1, 27638,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 13929,     0,  ..., 13929, 20301,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [   0,    0,    0,  ...,    0,    0,    0],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,    1,   31,    1],\n",
      "        [   1,    1,    1,  ...,    1, 2454,    1],\n",
      "        [   1,    1,    1,  ...,    1,    4,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [24513,     0,     0,  ..., 23087,  4890,  9415],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ..., 21683,     1,     1],\n",
      "        [    1,     1,     1,  ..., 20343,     1,     1],\n",
      "        [    1,     1,     1,  ...,     4,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [21854, 10904,  5837,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [26220,     0,     0,  ..., 14886,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [1071,    0,    0,  ...,    0,    0,    0],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [16604,     0,     0,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [12109, 17754,  5289,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     5,     1],\n",
      "        [    1,     1,     1,  ...,     1,  9334,     1],\n",
      "        [    1,     1,     1,  ...,     1,    27,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,     0,  ...,  9337,     0, 12927],\n",
      "        ...,\n",
      "        [    1,     1,     9,  ...,     1,     1,     1],\n",
      "        [    1,     1,   209,  ...,     1,     1,     1],\n",
      "        [    1,     1,     4,  ...,     1,     1,     1]])\n",
      "tensor([[   50,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [ 3923, 20060,     0,  ...,     0,  3178,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[ 49,  49,  49,  ...,  49, 170,  49],\n",
      "        [  7,   7,   7,  ...,   7,   7,   7],\n",
      "        [  0,   0,   0,  ...,   0, 203,   0],\n",
      "        ...,\n",
      "        [  1,   1,   1,  ...,   1,   1,   1],\n",
      "        [  1,   1,   1,  ...,   1,   1,   1],\n",
      "        [  1,   1,   1,  ...,   1,   1,   1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,     0,  ...,     0, 23581, 23854],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,  5064,     1],\n",
      "        [    1,     1,     1,  ...,     1,     2,     1],\n",
      "        [    1,     1,     1,  ...,     1,     0,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,     0,  ...,  6488,     0, 21005],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,     8,  ...,    49,    49,    49],\n",
      "        [    7,     7,     0,  ...,     7,     7,     7],\n",
      "        [ 3068, 21852,     2,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1, 17321,     1],\n",
      "        [    1,     1,     1,  ...,     1,     4,     1],\n",
      "        [    1,     1,     1,  ...,     1,    24,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 16540,     0,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [   0,    0, 4372,  ...,    0, 4461,    0],\n",
      "        ...,\n",
      "        [   1,    1,   12,  ...,    1,    1,    1],\n",
      "        [   1,    1,    6,  ...,    1,    1,    1],\n",
      "        [   1,    1, 2543,  ...,    1,    1,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,     0,  ...,     0, 16423,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,  4554,  ...,     0, 26249,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,     0,  ..., 14958,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 18875,   298,  ...,     0,     0, 12951],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1, 27424,     1],\n",
      "        [    1,     1,     1,  ...,     1,     3,     1],\n",
      "        [    1,     1,     1,  ...,     1,  1753,     1]])\n",
      "tensor([[  53,   49,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [ 103, 7634,    0,  ..., 6435,    0,    0],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1]])\n",
      "tensor([[  210,    49,    49,  ...,    49,    49,    50],\n",
      "        [   11,     7,     7,  ...,     7,     7,     7],\n",
      "        [  143,     0,     0,  ..., 20094,     0, 13870],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[49, 49, 49,  ..., 49, 49, 49],\n",
      "        [ 7,  7,  7,  ...,  7,  7,  7],\n",
      "        [ 0,  0,  0,  ...,  0,  0,  0],\n",
      "        ...,\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1],\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1],\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1]])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 24513,     0,  ...,  2796, 12384, 29233],\n",
      "        ...,\n",
      "        [  796,     1,     1,  ...,     1,     1,     1],\n",
      "        [  796,     1,     1,  ...,     1,     1,     1],\n",
      "        [20451,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [  616,     0, 12580,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,  6020,  3394,  ...,     0, 22188,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   50,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [5289,   72,    0,  ..., 5597,    0,    0],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,  9146, 25846,  ...,     0,     0,  2502],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,    11],\n",
      "        [    1,     1,     1,  ...,     1,     1,  2908],\n",
      "        [    1,     1,     1,  ...,     1,     1,    14]])\n",
      "tensor([[49, 49, 49,  ..., 49, 49, 49],\n",
      "        [ 7,  7,  7,  ...,  7,  7,  7],\n",
      "        [ 0,  0,  0,  ...,  0,  0,  0],\n",
      "        ...,\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1],\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1],\n",
      "        [ 1,  1,  1,  ...,  1,  1,  1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,  8057,     0,  ..., 22188,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   49,   49,  ...,   50,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [   0,    0,    0,  ...,   72,    0, 2502],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0, 14823,  ..., 11687,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   50,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [   72,     0,     0,  ..., 26611, 19098,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [29239, 12490,     0,  ..., 17684,     0,     0],\n",
      "        ...,\n",
      "        [    1,  1082,     1,  ...,     1,     1,     1],\n",
      "        [    1,   322,     1,  ...,     1,     1,     1],\n",
      "        [    1,     4,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 29545,     0,  ...,     0, 18880,  3178],\n",
      "        ...,\n",
      "        [    1,    33,     1,  ...,     1,     1,     1],\n",
      "        [    1,    52,     1,  ...,     1,     1,     1],\n",
      "        [    1,     0,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [16604,  5597,   899,  ...,     0, 16607,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 25771,  9369,  ..., 10253,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 14368, 23581,  ...,  2502,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 29545,   511,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,    33,     1,  ...,     1,     1,     1],\n",
      "        [    1,    52,     1,  ...,     1,     1,     1],\n",
      "        [    1,     0,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,     0,  ...,     0, 12109, 10259],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1, 26172],\n",
      "        [    1,     1,     1,  ...,     1,     1,   552],\n",
      "        [    1,     1,     1,  ...,     1,     1, 10259]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [13576,  9415,  9135,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ..., 15322,     1,     1],\n",
      "        [    1,     1,     1,  ...,     0,     1,     1],\n",
      "        [    1,     1,     1,  ...,  3699,     1,     1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [  24,    0,    0,  ...,    0,    0,    0],\n",
      "        ...,\n",
      "        [   1,    1,    0,  ...,    1,    1,    1],\n",
      "        [   1,    1,    2,  ...,    1,    1,    1],\n",
      "        [   1,    1, 2536,  ...,    1,    1,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,  7634,     0,  ...,     0,     0, 29555],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [16540,  4928,     0,  ..., 29702, 23581,  9304],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 12565,  6963,  ...,  4461,     0,  4554],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0, 14330,     0,  ...,  9321,  7748,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [   0,    0,    0,  ..., 4890,    0, 9928],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0, 13649,  ..., 17684,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,    16,     1,     1],\n",
      "        [    1,     1,     1,  ...,  3086,     1,     1],\n",
      "        [    1,     1,     1,  ...,   104,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [ 5811,     0, 23907,  ...,     0,     0,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [   0,    0,    0,  ...,    0,    0, 1615],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [    0,     0,     0,  ...,     0, 11741,     0],\n",
      "        ...,\n",
      "        [   27,     1,     1,  ...,     1,     1,     1],\n",
      "        [   16,     1,     1,  ...,     1,     1,     1],\n",
      "        [28452,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [   0, 4928,    0,  ...,    0,    0,    0],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,  261,    1,    1],\n",
      "        [   1,    1,    1,  ...,  580,    1,    1],\n",
      "        [   1,    1,    1,  ...,    4,    1,    1]])\n",
      "tensor([[  49,   49,   49,  ...,   49,   49,   49],\n",
      "        [   7,    7,    7,  ...,    7,    7,    7],\n",
      "        [   0, 9300, 9415,  ...,    0,    0,    0],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1],\n",
      "        [   1,    1,    1,  ...,    1,    1,    1]])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [18218,     0,     0,  ..., 20301,     0, 12174],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n",
      "tensor([[   49,    49,    49,  ...,    49,    49,    49],\n",
      "        [    7,     7,     7,  ...,     7,     7,     7],\n",
      "        [14194,     0,     0,  ..., 20610, 26481,     0],\n",
      "        ...,\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1],\n",
      "        [    1,     1,     1,  ...,     1,     1,     1]])\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-20d439a14c07>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mclassifier\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCNNClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEMBEDDING_DIM\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mFILTER_SIZES\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNUM_FILTERS\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mVOCAB_SIZE\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNUM_CLASSES\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mtest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclassifier\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSTATE_PATH\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_iter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_batches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_names\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-15-0cfa6d158348>\u001b[0m in \u001b[0;36mtest\u001b[0;34m(model, state_path, test_iter, batch_size, num_batches, target_names)\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mpredictions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcorrect\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtest_iter\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torchtext/data/iterator.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    155\u001b[0m                     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    156\u001b[0m                         \u001b[0mminibatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_key\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 157\u001b[0;31m                 \u001b[0;32myield\u001b[0m \u001b[0mBatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mminibatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    158\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrepeat\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m                 \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torchtext/data/batch.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, dataset, device)\u001b[0m\n\u001b[1;32m     32\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mfield\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m                     \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m                     \u001b[0msetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfield\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mclassmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torchtext/data/field.py\u001b[0m in \u001b[0;36mprocess\u001b[0;34m(self, batch, device)\u001b[0m\n\u001b[1;32m    199\u001b[0m         \"\"\"\n\u001b[1;32m    200\u001b[0m         \u001b[0mpadded\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 201\u001b[0;31m         \u001b[0mtensor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumericalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpadded\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    202\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torchtext/data/field.py\u001b[0m in \u001b[0;36mnumericalize\u001b[0;34m(self, arr, device)\u001b[0m\n\u001b[1;32m    298\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_vocab\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    299\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msequential\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 300\u001b[0;31m                 \u001b[0marr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstoi\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mex\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mex\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    301\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    302\u001b[0m                 \u001b[0marr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstoi\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torchtext/data/field.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    298\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_vocab\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    299\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msequential\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 300\u001b[0;31m                 \u001b[0marr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstoi\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mex\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mex\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    301\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    302\u001b[0m                 \u001b[0marr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstoi\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torchtext/data/field.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    298\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_vocab\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    299\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msequential\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 300\u001b[0;31m                 \u001b[0marr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstoi\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mex\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mex\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    301\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    302\u001b[0m                 \u001b[0marr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstoi\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "STATE_PATH = best_state_path\n",
    "num_batches = int(len(test_data) / BATCH_SIZE)\n",
    "\n",
    "classifier = CNNClassifier(EMBEDDING_DIM, FILTER_SIZES, NUM_FILTERS, VOCAB_SIZE+2, NUM_CLASSES)  \n",
    "\n",
    "test(classifier.to(device), STATE_PATH, test_iter, BATCH_SIZE, num_batches, target_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}