mercari-language-model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.callbacks.tracker import *\n",
    "from fastai.text import *\n",
    "from fastai.tabular import *\n",
    "from fastai_tab_text import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reset_seed(seed=42):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "#     tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    if torch.cuda.is_available(): \n",
    "        torch.cuda.manual_seed(seed)\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "reset_seed()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "mercari_path = Path('data/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_text_cols(df: pd.DataFrame) -> pd.DataFrame:\n",
    "  \n",
    "    df['category_name'] = df['category_name'].fillna('//')\n",
    "    df['category1'] = df['category_name'].apply(lambda x : x.split('/')[0].strip())\n",
    "    df.loc[df.category1=='','category1']= np.NaN\n",
    "    df['category2'] = df['category_name'].apply(lambda x : x.split('/')[1].strip())\n",
    "    df.loc[df.category2=='','category2']= np.NaN\n",
    "    df['category3'] = df['category_name'].apply(lambda x : x.split('/')[2].strip())\n",
    "    df.loc[df.category3=='','category3']= np.NaN\n",
    "    df['category_name'] = df['category_name'].apply( lambda x : ' '.join( x.split('/') ).strip() )\n",
    "    df.loc[df.category_name=='','category_name']= 'No category' # let this info in when concatenating text for RNN\n",
    "    \n",
    "    df_bn_fillna = df['brand_name'].fillna('No brand')\n",
    "    df['text'] = (df['name'].fillna('No name') + '. ' + df_bn_fillna + '. ' + \n",
    "                  df['category_name'] + '. ' + df['item_description'].fillna('No description'))\n",
    "    return df[['category1','category2','category3','brand_name', 'text', 'shipping', 'item_condition_id','price']]\n",
    "\n",
    "def preprocess_all(sample=None):\n",
    "    train = pd.read_table(mercari_path/'train.tsv').drop('train_id',axis=1)\n",
    "    price = train.price.values\n",
    "    train=train.drop('price',axis=1)\n",
    "    train['price']=price\n",
    "    \n",
    "    test = pd.read_table(mercari_path/'test_stg2.tsv').drop('test_id',axis=1)\n",
    "    test['price'] = np.NAN\n",
    "    train = train[train['price'] > 0].reset_index(drop=True)\n",
    "    all_df = pd.concat([train,test],axis=0).reset_index(drop=True)\n",
    "    del train\n",
    "    del test\n",
    "    gc.collect()\n",
    "#     if sample:\n",
    "#         np.random.seed(42)\n",
    "#         sample = np.random.permutation(sample)\n",
    "#         train = train.loc[sample].reset_index(drop=True)\n",
    "\n",
    "    all_df = preprocess_text_cols(all_df)\n",
    "    \n",
    "    return all_df\n",
    "def preprocess_train(sample=None):\n",
    "    train = pd.read_table(mercari_path/'train.tsv').drop('train_id',axis=1)\n",
    "    price = train.price.values\n",
    "    train=train.drop('price',axis=1)\n",
    "    train['price']=price\n",
    "\n",
    "    if sample:\n",
    "        np.random.seed(42)\n",
    "        sample = np.random.permutation(sample)\n",
    "        train = train.loc[sample].reset_index(drop=True)\n",
    "\n",
    "    train = preprocess_text_cols(train)\n",
    "\n",
    "    return train\n",
    "def get_val_idxs(train,n_splits=20):\n",
    "    np.random.seed(42)\n",
    "    cv = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    train_idxs, valid_idxs = next(cv.split(train))\n",
    "    return train_idxs,valid_idxs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "n=1482535 # train shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(222380, 8)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = preprocess_train(int(.15*n))\n",
    "train_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Language model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm = (TextList.from_df(train_df[['text']],path=mercari_path)\n",
    "                    .split_by_rand_pct(0.1)\n",
    "                   .label_for_lm()\n",
    "                   .databunch(bs=128)\n",
    "          )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm.save('data_lm.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm = load_data(mercari_path, 'data_lm.pkl', bs=60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.3).to_fp16()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4VOXd//H3d7LvCwQIYQm7IDtxAZdKtVqrVlHbamvd2lrXrj5tf09bn7Z20a5qrVVrazdtn8etbi1urTuigILIvhMIJCRk3zP374+ZxBADBJgzZyb5vK5rLmbOnJnzSZjkm3s59zHnHCIiIgABvwOIiEjsUFEQEZEuKgoiItJFRUFERLqoKIiISBcVBRER6aKiICIiXVQURESki4qCiIh0SfQ7wKEaPHiwKy4u9juGiEhcWbp06R7nXMHB9ou7olBcXMySJUv8jiEiElfMbGtf9lP3kYiIdFFREBGRLioKIiLSRUVBRES6qCiIiEgXFQUREemioiAiIl1UFERE4sDtz6/nlfUVnh/H06JgZrlm9rCZrTGz1WY2dz/7HWNmHWZ2oZd5RETikXOOO/69njc2VXp+LK/PaL4dWOicu9DMkoH0njuYWQJwK/CMx1lEROJSU1sHHUFHVmqS58fyrKVgZtnAycDvAZxzrc656l52vQF4BCj3KouISDyra24HICvV+5WJvOw+GgtUAPeb2dtmdp+ZZXTfwcyKgAXA3R7mEBGJa3XNbQDx3VIg1DU1G/itc24W0AB8q8c+twHfdM51HOiNzOwqM1tiZksqKrwfaBERiSW1/aSlUAqUOucWhx8/TKhIdFcC/N3MtgAXAneZ2Xk938g5d69zrsQ5V1JQcNCVX0VE+pWu7qMU74uCZ0dwzu0ys+1mNsk5txY4FVjVY58xnffN7I/AU865f3iVSUQkHtV3tRS87z7yuuzcADwQnnm0CbjCzK4GcM5pHEFEpA/eH1OI45YCgHPuHUJdRN31Wgycc5d7mUVEJF71l9lHIiISAXXNbZhBRrKKgojIgFfb3E5mSiKBgHl+LBUFEZEYV9fcHpWZR6CiICIS8+pb2qIy8whUFEREYl5dc3tUBplBRUFEJOapKIiISJe6ZnUfiYhIWF1zO5lqKYiICEBdi7qPREQEaGnvoLU9SLa6j0REJJpLXICKgohITFNREBGRLl0rpKao+0hEZMDrbClo9pGIiKj7SERE3tfZfaTZRyIiopaCiIi8r2tMQUtni4hIXXMbaUkJJCZE59e1ioKISAyL5gqpoKIgIhLT6qO47hGoKIiIxLTaKC6bDRC98hPDapra+N4T77F5TwO1zW3UNrWRkpjAlSeO4TPHjSI1KcHviCIyQEW7+2jAF4Wm1g4+98e3WF5azfFjB1GUl0Z2ahKbKuq5+alV3PPSRq49ZRzzjxpCR9ARdI7EQIDRg9IxM7/ji0g/V9fcRlFuWtSON6CLQmt7kKv/upRl2/by64tnc9b0wn2eX7Sxkl89v47vPbmK7z25ap/njhuTz7fOPIpZo/KiGVlEBpi65vaoTUeFAVQU2juCBB0kJ4aGUTqCjq/+7zu8tK6CWy+Y9oGCADB33CCOH3s8S7fuZVtVIwkBI2DGrppm7nl5Iwvuep0zpw7jmlPGMXFolrqZRCTi1H3kkedW7eaaB5aRnBAgIyU057eiroVvf2wynzpm1H5fZ2aUFOdTUpy/z/ZPHzeK372yiXtf3sS/Vu7CDIbnpFE8OJ3MlEQCFiogDkdja0f41k57h8PMCBgkBIyctCTyM5IZlJFCQVYKI/PTGJWfzqj8dHLSktRFJTKAtXcEaWrr0ECzFyYMzeTG0ydS39JBQ0s7DS3tzCnO4zPHjT6s98tISeQrp03kkuNH89qGPWzZ08iWygY272lgT10rQedw4X3TkxNIS0pgSFYqiQHDAc452oOO6sY2tlY2UlnfQkNrxz7HGJmfxvxJQ5g/aQhzxw36QEukvSPIih01vLGpkoAZQ7NTGJqVSkFWCjnpSWSnJqn1IhLH6luiu8QFDKCiMH5IFtd/OCvi7zs4M4VzZxZF5L3qW9rZXtXItqpGtlU2snhzJQ8tKeXPi7aSnBBgeG4qhTlpFOam0tjSwesb91AbPgV+f1ISA0wuzOaS40dz9vTCfYpER9Cxq7aZ8tpmyutaqKhrISUxQFFuGsNz0xiWk6qiIuKjaK97BAOoKMSDzJREJhdmM7kwG4AvnDyW5rYOFm+u4vWNeyjd20RZdROLNoZaBh+dOoyTJhRwwvjBJCcGKK9tZndtC+V1zdQ2t1Pb1EZ1Yyv/WVvBjQ8t54dPr2LBrCJa2oOsLqtlTVkdTW0dB8w0LDs11J01KJ1jivO4cM5IEgLq0hKJhtrOC+yoKEin1KQEPjSxgA9NLDjovpkFmYwtyPzA9v/+mGPRpkr+smgrf160lfTkBKYUZvOpY0YyYWgmw7JTGZKVypDsFFraguyobmJndROle5tCrZaqBl5aV8HDS0v56xvb+NGCqUwfkevFlysi3bzfUtCYgkSQmTFv3GDmjRtMc1sHKYmBAw5gjxqU/oFtzjmeWlHGD55axXm/eY1L5xZzyfGjGTM4Qy0HEY/Uq/tIvHa4YwRmxjkzhnPyxAJ+/sxa/rRoC398fQspiQEmDctiSmE2s0flMXt0HmMHZxAIF4qm1g72NrZSkJVCUpRWeRTpL+paOruP1FKQGJWTlsTN503lihOKWbatmtVltawuq+VfK3fx97e2A5CbnkR+ejIVdS3UhWdPJCUY4woyOWpYFsNy0qhubGVPfStVDS0U5qbxoQkFnDyxgGE5qYecKRh0VDa0khAw8tI1jVf6Dw00S9wY22P8Ihh0bNrTwLKte1m6dS/1re2cnBk69yI3PYntVU2s3VXL4s1VVNS1hM7NyEwhLz2JtzZX8fSKMgAmDMlk5shcpo3IYWpRDkW5aTS2hqYR17e0s7O6ia2VoRla26saKatppryumbaO0ATgzJRERuanU5SbSnNbkMqGVvY2tNIeDDIiL3T+x+hB6aQnJxJ0jvYOR0IAZo/KY05xHimJmm0lsaPfFQUzywXuA6YCDrjSObeo2/OfAb4ZflgPXOOcW+5lJvFGIGCMH5LJ+CGZfPKYkQfc1zm3z1/zzjnW7q7j5XUVvLahkhfWlPPQ0tL9vt4MCrNTGZmfzrFj8hmWk8qw7FQ6gq6rWJTubSI9OYGi3FSmFWUTMKN0bxNvb9/LUyt2EnQffN/UpADHjRnE0cOzSQwYZkZiwJgzOo/jxg7S2IlEXW1zG8kJgaj+seJ1+bkdWOicu9DMkoGeI5ibgQ855/aa2ZnAvcBxHmcSn/Xs3jEzjhqWzVHDsrnq5HE45yiraebdHTXsqW8hIzmR9OQEMlISGZqdysj8tCP6IWnrCIZbCEZCwGhsbWfxpipe3bCHV9ZX8NqGPQSd26dwDMtO5dxZwzl72nAmDstUi0KiItpLXICHRcHMsoGTgcsBnHOtQGv3fZxzr3d7+AYwwqs8Ej/MjOHhE+i8kJQQoPt4e1ZqEqdNGcppU4bus59zjua2IM+v3s0/3t7B71/ZzD0vbSJgMCo/nXEFmcwpzuPcmUVRXcVSBo76/lQUgLFABXC/mc0AlgJfds417Gf/zwH/6u0JM7sKuApg1Kj9r1MkEklmRlpyAufMGM45M4ZTWd/Cqxv2sLG8no0VDawvr+OFNeX87Jm1HDcmnwWzipgzOo/RgzI000oioi7KF9gBb4tCIjAbuME5t9jMbge+BXy3545mNp9QUTixtzdyzt1LqGuJkpKSXnqDRbw3qJclTbZVNvKPd3bw6LJSvvnIu0BoptWYwRlMK8rlU8eM5JjiPM2IksPSr7qPgFKg1Dm3OPz4YUJFYR9mNp3QYPSZzrlKD/OIRNyoQel86dQJ3PDh8awuq2PNrlrWl9ezfncdz67axSPLShk/JJNPHzuKC0tGkB3lv/okvtU1t1M8+IMnk3rJs6LgnNtlZtvNbJJzbi1wKrDPlWrMbBTwKPBZ59w6r7KIeM3MmDI8mynDs7u2NbV28OSKnTyweBs/eGoVt7+wni9+aCyXzysmPVmzweXg6prbyEzpP91HADcAD4RnHm0CrjCzqwGcc3cDNwGDgLvCzet251yJx5lEoiItOYFPlozkkyUjWb69mtueX8dPF67lD69u5rr547l0brGmucoB9bfuI5xz7wA9f8nf3e35zwOf9zKDSCyYMTKX+684lqVbq/j5M+v4/pOrWLypitsumqnlyaVXwaCjvrWd7CgXBU2REImiOaPz+dtVx3PT2VN4ZtUuLv39m9Q0tvkdS2JQQ2s7zkV33SNQURDxxZUnjuHXF8/ine3VXHj36+ysbvI7ksQYP5a4ABUFEd+cPX04f7zyGHbVNHPub15j0UZNvpP3dRaFTBUFkYFj3rjBPHzNPLJSE/nMfW/wm/9sINjbwkwy4NQ1R3/ZbFBREPHdpGFZPHH9iZw9fTg/e2YtV/zxLSrrW/yOJT7rXHZe3UciA1BmSiK3XzSTHy2YyqJNlXzsjlfUnTTAdXYfafaRyABlZnzmuNE8du08MpJD3Um/em4dHepOGpDUfSQiABw9PIcnbziR82YVcfsL6/n0796gvK7Z71gSZZp9JCJdMlIS+eUnZ/KLT8xgRWkNZ9/xKku3VvkdS6KorrmNhICRFuWTG1UURGLYBXNG8Oi180hLTuCie9/gL4u24Jy6kwaCuuZ2MlMSo77CroqCSIybXJjNE9edyInjB/Pdx9/j/z36Lu0dQb9jicf2NraRlx79VXVVFETiQE56Er+/7Biunz+ev7+1nav/uozmtg6/Y4mHqhpaGJSZEvXjqiiIxIlAwLjxjEn84NyjeWHNbj77+8VaN6kfq6xvJT8jOerHVVEQiTOXzi3uWjfpk/csoqJOJ7r1R5UNrQxSURCRvjh7+nDuv/xYtlY1cN2Dy2jTGEO/Egw6qhpaGZSpoiAifXTihMH85PxpvLm5ilv+tcbvOBJBtc1tdAQd+RnRH1PQNQFF4tiCWSNYvr2G37+6mRkjc/n4jOF+R5II2FPfCsBgtRRE5FD998cmUzI6j28+vIK1u+r8jiMRUNUQKgoaaBaRQ5acGOCuz8wmMzWRL/5lCfXh1TUlflU1hCYPqCiIyGEZkp3KnRfPYltVIzc9vtLvOHKE3u8+0nkKInKYjhs7iBs+PIFHl+3gsbdL/Y4jR6Cz+ygvXS0FETkCN3x4PMcU5/Gdx1aytbLB7zhymCrrW8hOTSQ5Mfq/olUURPqRxIQAt100i4SA8aW/vU1ru85fiEeVDa2+LHEBKgoi/U5Rbhq3XjCd5aU1On8hTlX5dDYzqCiI9EtnTivk8nnF/OG1zfz9zW1+x5FD5Ne6R6CiINJvfeesyZw0YTDf+cdKXe85zlT6tMQFqCiI9FuJCQHu/PRsRg9K55oHlrJljwae40Ew6Njb2MogH5a4ABUFkX4tJy2JP1x+DACf+9Nb1DZrqe1YV9PUue6RWgoi4oHRgzK4+5I5bK1s5Gv/+w7BoC7nGcsqw2czq/tIRDxz/NhBfOesyTy/upw7/7PB7zhyAJXhs5nVfSQinrpsXjELZhXxq+fX8Z815X7Hkf3oPJtZLQUR8ZSZ8eMF05g8LJsv//1tnfEco/Z0FgWNKYiI19KSE7jns3MwM659QFdsi0VV4e6jPBUFEYmGkfnp/PTC6by3s5bfvrjR7zjSQ2VDCzlpSSQl+PPr2dOjmlmumT1sZmvMbLWZze3xvJnZHWa2wcxWmNlsL/OISMgZRw/jnBnD+fW/17NmV63fcaSbSh+XuADvWwq3Awudc0cBM4DVPZ4/E5gQvl0F/NbjPCIS9v2PH012ahLfeHgF7epGihmV9S2+DTKDh0XBzLKBk4HfAzjnWp1z1T12Oxf4swt5A8g1s0KvMonI+/IzkvnBuVNZUVrD717Z7HccCatq8G/dI+hjUTCzcWaWEr5/ipl9ycxyD/KysUAFcL+ZvW1m95lZRo99ioDt3R6XhreJSBR8bNowPnr0MH71/Do2lOv6zrGgysdls6HvLYVHgA4zG0/oL/8xwIMHeU0iMBv4rXNuFtAAfKvHPtbL6z5wuqWZXWVmS8xsSUVFRR8ji8jBmBk3nzeV9OQEvv6QupH8Fgw6X5fNhr4XhaBzrh1YANzmnPsqcLBunlKg1Dm3OPz4YUJFouc+I7s9HgHs7PlGzrl7nXMlzrmSgoKCPkYWkb4oyErh5nOnsnx7Nfe8vMnvOANadVMbQUfsdx8BbWZ2MXAZ8FR4W9KBXuCc2wVsN7NJ4U2nAqt67PYEcGl4FtLxQI1zrqyPmUQkQs6ZMZyzphVy2/PrWF2m2Uh+qazvXPco9ruPrgDmAj9yzm02szHAX/vwuhuAB8xsBTAT+LGZXW1mV4ef/yewCdgA/A649pDSi0jE3HzeVHLSkvja/y3XZTx9Uunz2cwQ6vc/KOfcKuBLAGaWB2Q5527pw+veAUp6bL672/MOuK7PaUXEM/kZyfx4wTSu+stS7vz3er52+qSDv0giyu91j6Dvs49eNLNsM8sHlhOaUfRLb6OJSLSdfvQwzp9dxG9e3KiT2nzQ2X0UD2MKOc65WuB84H7n3BzgNO9iiYhfvnvWFLJSE/n+E6sINeYlWjq7j/LTY78oJIZPKvsk7w80i0g/lJeRzNdPn8SiTZX8891dfscZUCrrW8lNTyLRp3WPoO9F4QfAM8BG59xbZjYWWO9dLBHx06ePHcXkwmx+9PQqmlo7/I4zYPh9NjP0sSg45x5yzk13zl0TfrzJOXeBt9FExC8JAeN750xhZ00zv31JK6lGy576Fgb7dMW1Tn0daB5hZo+ZWbmZ7TazR8xshNfhRMQ/x40dxDkzhnP3SxvZXtXod5wBIW5aCsD9hE40G05obaInw9tEpB/7748dRYIZ339Sg87REFr3KD6KQoFz7n7nXHv49kdA602I9HOFOWl85bQJPL96NwtXatDZSx1BR1Wjv+seQd+Lwh4zu8TMEsK3S4BKL4OJSGz43IljmFKYzf888R41TW1+x+m3qhtbcT6vewR9LwpXEpqOugsoAy4ktPSFiPRziQkBbr1gOnvqW7h14Rq/4/RbXUtc+LjuEfR99tE259zHnXMFzrkhzrnzCJ3IJiIDwLQROVx5whgeXLyNt7ZU+R2nXyqvDZ3NPDgeisJ+fC1iKUQk5n31IxMpyk3jW4+soKVd5y5E2s6aJgCG56b6muNIikJvF8gRkX4qIyWRHy6YysaKBh5cvM3vOP1OWXUzAMNy4rcoaH6ayAAzf9IQjh2Tz70vb9Ly2hFWVtPE4MwUUhITfM1xwKJgZnVmVtvLrY7QOQsiMsBcP388ZTXNPLqs1O8o/crOmmbfu47gIEXBOZflnMvu5ZblnOvTtRhEpH85acJgphXl8NuXNuqazhFUVt1Eoc9dR3Bk3UciMgCZGdfNH8/WykaefldXz42UsppmCnPS/I6hoiAih+70KUOZMCSTu/6zkWBQw4tHqra5jfqW9tjvPhIR6U0gYFw7fxxrd9fxwppyv+PEvZ3VoemoaimISNw6Z/pwRuan8et/r1dr4Qh1TkdVS0FE4lZiQoAvnzqRFaU1/O0tnbdwJDpPXFNLQUTi2gWzi5g3bhC3/HMNu2ub/Y4Tt8qqm0kIGEOy/F3iAlQUROQImBk/XjCN1o4gNz2+0u84cWtnTRNDs1J8vTZzJ/8TiEhcKx6cwVdOm8gz7+1m4UpNUT0cZdXNFOb633UEKgoiEgGfP2kMkwuzuelxXXPhcJTVxMaJa6CiICIRkJQQ4NYLprGnvoVfPbfO7zhxxTkXXuJCLQUR6Uemj8jlkyUjeXDxNnaE593LwVU2tNLaHlRLQUT6nxtOnQDAnf9e73OS+NF5jkIsTEcFFQURiaCi3DQuOnYkDy0pZWtlg99x4kKsXFynk4qCiETUdfPHkxAwbn9BrYW+KIuhJS5ARUFEImxodiqXzh3NP97ewYbyer/jxLyymmaSEwMMykj2OwqgoiAiHrj6Q+NITUrgtuc1E+lgdtY0U5iTSiAQG1c4VlEQkYgblJnCFScU89SKMtbsqvU7TkyLlYvrdFJREBFPfOGksWSmJHKHxhYOaGd1E8NjZDwBVBRExCO56clcPq+Yf767i7W76vyOE5M6go7ddS0UxsjMI/C4KJjZFjN718zeMbMlvTyfY2ZPmtlyM3vPzK7wMo+IRNfnThxDRnICd+i8hV6V1zXTEXQxM/MIotNSmO+cm+mcK+nlueuAVc65GcApwC/MLDaG4EXkiOVlJHP5CcX8890y1u1Wa6GnnTF0cZ1OfncfOSDLzAzIBKqAdn8jiUgkff7EsaQnJWhsoRdlMXRxnU5eFwUHPGtmS83sql6evxOYDOwE3gW+7JwLepxJRKIoLyOZS+cV8/S7ZaxXa2EfXZfhHEBF4QTn3GzgTOA6Mzu5x/NnAO8Aw4GZwJ1mlt3zTczsKjNbYmZLKioqPI4sIpH2hZPGkpaUwB3/3uB3lJiys6aJjOQEstMS/Y7SxdOi4JzbGf63HHgMOLbHLlcAj7qQDcBm4Khe3ude51yJc66koKDAy8gi4oH8jNBMpCeX72Tljhq/48SMzovrhHrQY4NnRcHMMswsq/M+cDrQ83p924BTw/sMBSYBm7zKJCL++eKHxpGbnsRPn1nrd5SYsTOGLq7TycuWwlDgVTNbDrwJPO2cW2hmV5vZ1eF9bgbmmdm7wAvAN51zezzMJCI+yUlL4vr543l5XQWvbdCPuXOOLXsaGJWf7neUfXjWkeWc2wTM6GX73d3u7yTUghCRAeCS40dz/2tbuOVfa3j8uhNiZr0fP1TUt1Db3M74IZl+R9mH31NSRWQASU1K4Ksfmci7O2r458oyv+P4asPu0AqyE4Zk+ZxkXyoKIhJVC2YVMWloFj9/Zi1tHQN3BvqGilBRUEtBRAa0hIDxzTMnsaWykYeWlPodxzfrd9eTlZLI0OwUv6PsQ0VBRKJu/qQhzBiRw70vb6Qj6PyO44sN5fWMG5IZU9NRQUVBRHxgZlx18ji2VDby3KrdfsfxxYaK+pjrOgIVBRHxyRlHD2Vkfhr3vrzR7yhRV9PYRkVdCxNUFEREQhITAnz+xLEs21bNki1VfseJqg0VoTWg1FIQEenmEyUjyE1P4t6XB9ZCBhvKY3M6KqgoiIiP0pMTufT40Ty3ejebwlM0B4L1u+tJSQxQlBc7q6N2UlEQEV99dm4xSQkBfvfKZr+jRM2GinrGFmSSEINndKsoiIivCrJSuGD2CB5ZVsr2qka/40TF+t31MTnIDCoKIhIDbvjweBIDxncfX4lz/fu8hcbWdnZUN8XkIDOoKIhIDBiem8bXT5/Ei2sreGpF/14TaVNFA4BaCiIiB3L5vGKmFeXw/SdXUdPU5nccz6wvj93pqKCiICIxIiFg/OT8aVQ1tHDrwjV+x/HMhvJ6EgPG6EEZfkfplYqCiMSMqUU5XHHCGB5cvK3fntC2obye0YPSSU6MzV+/sZlKRAasr31kIkW5adz0+HsE++FieevLY3PNo04qCiISUzJSEvnGRyexqqyWx5fv8DtORLW2B9la2RiTZzJ3UlEQkZhzzvThTC3K5ufPrKO5rcPvOBGzpbKBjqBTS0FE5FAEAsZ/nzmZHdVN/HnRFr/jREznmkcqCiIih2je+MGcMqmAO/+9gerGVr/jRMSasloCBuMKVBRERA7ZNz96FHUt7dz1Yv+45sKybdUcNSybtOQEv6Psl4qCiMSsyYXZXDB7BH98bQule+N7XaSOoOOd7dXMHp3rd5QDUlEQkZj2tY9MBIPbn1/vd5Qjsm53HfUt7cwZned3lANSURCRmDY8N43PHj+aR5aVsjGOr7mwbNteAGaPUlEQETki154yjrSkBH753Dq/oxy2pVv3MjgzmVH56X5HOSAVBRGJeYMyU7jyxDE8vaKMlTtq/I5zWJZt3cvsUXmYxd6FdbpTURCRuPD5k8aSk5YUl62FyvoWtlQ2MjvGxxNARUFE4kROWhJXf2gc/15TztKt8bVY3rJt1QAxP8gMKgoiEkcumzeawZkp3LpwbVxdoW3Ztr0kBoxpRTl+RzkoFQURiRvpyYl85bQJvLm5iifj6AptS7fu5eiiHFKTYvektU4qCiISVy4+dhTTR+Rw81OrqG2O/Su0tXUEWVFazexRsX3SWicVBRGJKwkB44fnTWVPfQu/fDb2B51Xl9XS3BaMi/EEUFEQkTg0fUQulxw3mj8v2hLzU1SXbo2Pk9Y6qSiISFy68YxJ5Gck851/rIzpK7Qt21ZNYU4qw3PT/I7SJ54WBTPbYmbvmtk7ZrZkP/ucEn7+PTN7ycs8ItJ/5KQl8e2zJvPO9mr+9tY2v+Ps17Kte+Pi/IRO0WgpzHfOzXTOlfR8wsxygbuAjzvnjgY+EYU8ItJPnDeziLljB3HLv9awu7bZ7zgfsGVPAzuqm5gTJ11H4H/30aeBR51z2wCcc+U+5xGROGJm/Pj8abS0B/mfx9/zO84HPPb2DszgzGnD/I7SZ14XBQc8a2ZLzeyqXp6fCOSZ2YvhfS71OI+I9DNjBmfwldMmsPC9XTzz3i6/43RxzvHo26WcMG4whTnxMZ4A3heFE5xzs4EzgevM7OQezycCc4CzgDOA75rZxJ5vYmZXmdkSM1tSUVHhcWQRiTdfOGkskwuzuenxlTFz7sKSrXvZXtXE+bOL/I5ySDwtCs65neF/y4HHgGN77FIKLHTONTjn9gAvAzN6eZ97nXMlzrmSgoICLyOLSBxKSghwy/nTqKhr4dZ/rfE7DgCPLislPTmBM46On64j8LAomFmGmWV13gdOB1b22O1x4CQzSzSzdOA4YLVXmUSk/5oxMpcrThjDA4u38cfXNvuapbmtg6dWlPHRqcPISEn0Ncuh8jLtUOCx8NrhicCDzrmFZnY1gHPubufcajNbCKwAgsB9zrmehUNEpE/+64xJbKtq5HtPrmJLZSPfPXsKCYHoX7/g+dW7qWtu54LZI6J+7CPlWVFwzm2i966gu3s8/hnwM69yiMjAkZqUwN2XzOEn/1zNfa9uZntVI3fakTj8AAAK8klEQVRcPCvqf60/umwHhTmpHD92UFSPGwl+T0kVEYmohIDxnbOncPN5U/nP2nI++/vFtLR3RO34FXUtvLSugnNnFvnSSjlSKgoi0i999vjR/Pri2SzbVs33nlgVteM+sXwnHUEXd7OOOqkoiEi/ddb0Qq49ZRx/e3Mbf3/T+6Uw2jqC/GXRFmaMyGHi0CzPj+cFFQUR6de+fvokTpowmJsef493tld7eqy/v7WdLZWNfPm0CZ4ex0sqCiLSryUEjDsumkVBVgrX/HUp5XXerJHU2NrOHS+s59jifOZPGuLJMaJBRUFE+r28jGTu+ewc9ja2suA3r/Pezshfg+EPr26moq6Fb545ifBU/LikoiAiA8LUohz+74tzCTrHhb9dxNMRvMbz3oZW7nlpE6dNHsqc0fkRe18/qCiIyIAxfUQuj19/AlOGZ3Pdg8v42TNr6IjABXruenEDDa3tfOOjkyKQ0l8qCiIyoAzJSuXBLxzHxceO5Df/2cjl979JVUPrYb/f9qpG/rRoK+fPHhG3M466U1EQkQEnJTGBn5w/nVsvmMbizVWcdccrLNu295Dfp7K+hcvuf5OUhABf/cgHFniOSyoKIjJgfeqYUTx6zTwSE4xP3bOI255fR01j35bermtu47L732TH3ibuu6yEoji5BvPBqCiIyIA2tSiHp64/iY9MGcptz69n3i0v8MOnVrGzumm/r2lu6+Dzf1rCmrI67r5kDsfF4RpH+2POHfkgSzSVlJS4JUuW+B1DRPqh1WW13PPSRp5cUYYBc8cN4oyjh3H6lKEMzkxhXXkdb23Zy+Nv72Dptr3c9qmZnDszPpazMLOlzrmSg+6noiAisq/tVY08sHgbC1eWsaWyETPITE6krqUdgCFZKdx4xiQ+WTLS56R9p6IgInKEnHOs213PwpW72F3XzOxReRxbnM/I/LS4O0Gtr0Uhvi4JJCISRWbGpGFZTBoW/1NN+0oDzSIi0kVFQUREuqgoiIhIFxUFERHpoqIgIiJdVBRERKSLioKIiHRRURARkS5xd0azmVUA1UDP6+nlHGTbwe53/jsY2HMY0Xo7fl+e77n9QI97Zu2+7XByRzNz9/t+fK/1+dDn40DPx+Pn41AyA0xwzuUcNIlzLu5uwL2Huu1g97v9uyRSmfryfM/tB3rcM+uR5o5mZr+/1/p86PPR3z4fh5K5L8fovMVr99GTh7HtYPd7e/2RZurL8z23H+hxb1mPJHc0M3e/78f3Wp+PQ6fPR9/vx3rmvhwDiMPuI6+Z2RLXh0WjYk085lbm6InH3Mrsj3htKXjpXr8DHKZ4zK3M0ROPuZXZB2opiIhIF7UURESkS78uCmb2BzMrN7OVh/HaOWb2rpltMLM7rNsVNczsBjNba2bvmdlPI5vam9xm9j0z22Fm74RvH4v1zN2ev9HMnJkNjlxiz77PN5vZivD3+FkzGx4HmX9mZmvCuR8zs9xIZvYw9yfCP4NBM4tYP/6RZN3P+11mZuvDt8u6bT/g5943hzN9Kl5uwMnAbGDlYbz2TWAuYMC/gDPD2+cDzwMp4cdD4iT394Ab4+l7HX5uJPAMsBUYHOuZgexu+3wJuDsOMp8OJIbv3wrcGg+fD2AyMAl4ESjxO2s4R3GPbfnApvC/eeH7eQf6uvy+9euWgnPuZaCq+zYzG2dmC81sqZm9YmZH9XydmRUS+uFe5EL/e38Gzgs/fQ1wi3OuJXyM8jjJ7SkPM/8K+AYQ8cEvLzI752q77ZoR6dweZX7WOdce3vUNYEQkM3uYe7Vzbm2sZN2PM4DnnHNVzrm9wHPAR/38WT2Yfl0U9uNe4Abn3BzgRuCuXvYpAkq7PS4NbwOYCJxkZovN7CUzO8bTtO870twA14e7CP5gZnneRe1yRJnN7OPADufccq+DdnPE32cz+5GZbQc+A9zkYdZOkfhsdLqS0F+t0RDJ3F7rS9beFAHbuz3uzB8rX9cHDKhrNJtZJjAPeKhb911Kb7v2sq3zL75EQs3A44FjgP8zs7Hhau+JCOX+LXBz+PHNwC8I/QLwxJFmNrN04NuEujaiIkLfZ5xz3wa+bWb/D7ge+J8IR30/SIQyh9/r20A78EAkM/Ymkrm9dqCsZnYF8OXwtvHAP82sFdjsnFvA/vP7/nXtz4AqCoRaRtXOuZndN5pZArA0/PAJQr9AuzehRwA7w/dLgUfDReBNMwsSWu+kIpZzO+d2d3vd74CnPMwLR555HDAGWB7+QRwBLDOzY51zu2I0c08PAk/jYVEgQpnDA6BnA6d6+QdON5H+Xnup16wAzrn7gfsBzOxF4HLn3JZuu5QCp3R7PILQ2EMp/n9dvfN7UMPrG1BMtwEj4HXgE+H7BszYz+veItQa6BwE+lh4+9XAD8L3JxJqGloc5C7sts9Xgb/HeuYe+2whwgPNHn2fJ3Tb5wbg4TjI/FFgFVAQ6azR+HwQ4YHmw83K/geaNxPqXcgL38/v6+fej5vvATz94uBvQBnQRqgyf47QX58LgeXhH4Sb9vPaEmAlsBG4k/dP9EsG/hp+bhnw4TjJ/RfgXWAFob/ACmM9c499thD52UdefJ8fCW9fQWitmaI4yLyB0B8374RvEZ0x5WHuBeH3agF2A8/4mZVeikJ4+5Xh7/EG4IpD+dz7cdMZzSIi0mUgzj4SEZH9UFEQEZEuKgoiItJFRUFERLqoKIiISBcVBekXzKw+yse7z8ymROi9Oiy0qupKM3vyYKuUmlmumV0biWOL9KQpqdIvmFm9cy4zgu+X6N5fJM5T3bOb2Z+Adc65Hx1g/2LgKefc1Gjkk4FFLQXpt8yswMweMbO3wrcTwtuPNbPXzezt8L+TwtsvN7OHzOxJ4FkzO8XMXjSzhy10vYEHOte8D28vCd+vDy+Ct9zM3jCzoeHt48KP3zKzH/SxNbOI9xcEzDSzF8xsmYXW3T83vM8twLhw6+Jn4X3/K3ycFWb2/Qh+G2WAUVGQ/ux24FfOuWOAC4D7wtvXACc752YRWsX0x91eMxe4zDn34fDjWcBXgCnAWOCEXo6TAbzhnJsBvAx8odvxbw8f/6Dr2oTX/TmV0BnnAM3AAufcbELX8fhFuCh9C9jonJvpnPsvMzsdmAAcC8wE5pjZyQc7nkhvBtqCeDKwnAZM6bayZbaZZQE5wJ/MbAKhlSmTur3mOedc97X033TOlQKY2TuE1sR5tcdxWnl/gcGlwEfC9+fy/hr5DwI/30/OtG7vvZTQmvsQWhPnx+Ff8EFCLYihvbz+9PDt7fDjTEJF4uX9HE9kv1QUpD8LAHOdc03dN5rZr4H/OOcWhPvnX+z2dEOP92jpdr+D3n9m2tz7g3P72+dAmpxzM80sh1BxuQ64g9D1GAqAOc65NjPbAqT28noDfuKcu+cQjyvyAeo+kv7sWULXMwDAzDqXPs4BdoTvX+7h8d8g1G0FcNHBdnbO1RC6hOeNZpZEKGd5uCDMB0aHd60Dsrq99BngyvC6/5hZkZkNidDXIAOMioL0F+lmVtrt9jVCv2BLwoOvqwgtew7wU+AnZvYakOBhpq8AXzOzN4FCoOZgL3DOvU1oJc6LCF3spsTMlhBqNawJ71MJvBaewvoz59yzhLqnFpnZu8DD7Fs0RPpMU1JFPBK+elyTc86Z2UXAxc65cw/2OhE/aUxBxDtzgDvDM4aq8fDypyKRopaCiIh00ZiCiIh0UVEQEZEuKgoiItJFRUFERLqoKIiISBcVBRER6fL/AYrHQI6Ke/P4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 10:26 <p><table style='width:375px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "    <th>time</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>0</th>\n",
       "    <th>3.018970</th>\n",
       "    <th>2.837136</th>\n",
       "    <th>0.493465</th>\n",
       "    <th>10:26</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 2e-2, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learn.save('bs60-awdlstm-stage1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('bs60-awdlstm-stage1');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XucXHV9//HXZ2Z29n7JZXMhF0ISQBEUkhWJoOKlKEJBKlZs+RUU5YG/eql9oL/ys0VLL1Ztq6XY8osoRZGioFigKqA0gsjFBJIQLoGQQK5kd7Obve/szszn98ecXYZlN9kke2bOzL6fj8c89syZM3M+c3Z33vP9nnO+x9wdERERgFixCxARkehQKIiIyCiFgoiIjFIoiIjIKIWCiIiMUiiIiMgohYKIiIxSKIiIyCiFgoiIjEoUu4BDNXv2bF+yZEmxyxARKSnr1q1rd/fmgy1XcqGwZMkS1q5dW+wyRERKipm9NJnl1H0kIiKjFAoiIjJKoSAiIqMUCiIiMkqhICIioxQKIiIySqEgIiKjFAoiIiXgm798jgefbwt9PQoFEZGIy2Sda3/1PI9t6wh9XQoFEZGI6+gbIuswu64y9HUpFEREIq69NwUoFEREhPxQSIa+LoWCiEjEjYZCvVoKIiLTXnvPEKDuIxERIddSSCZiNFSFf7UDhYKISMS19aRorqvEzEJfl0JBRCTi2npTBdnJDCGHgpm9aGZPmtl6M3vN5dLM7Ewz6woeX29mV4dZj4hIKWrvHSrI/gQozOU43+nu7Qd4/EF3P7cAdYiIlKT23hRvXNBYkHWp+0hEJMKyWaejb4jZ9WXQfQQ4cK+ZrTOzyydYZpWZbTCzn5vZG8ZbwMwuN7O1Zra2rS38AaFERKKis3+ITNbLpvvodHffbWZzgPvM7Fl3fyDv8ceBo92918zeD/wUOHbsi7j7amA1QEtLi4dcs4hIZLT35s5RaC7AiWsQckvB3XcHP1uBO4BTxzze7e69wfTPgAozmx1mTSIipaSQ4x5BiKFgZrVmVj8yDZwFbBqzzDwLDrw1s1ODevaFVZOISKlp6ylsKITZfTQXuCP4zE8At7j7L8zsCgB3vx64EPikmaWBAeAid1f3kIhIYKSl0FzqoeDuW4E3jTP/+rzp64DrwqpBRKTUtfWmSMZjNFQX4gwCHZIqIhJp7T1DzKpLFmSIC1AoiIhEWntvqmD7E0ChICISae0FHPcIFAoiIpHW3psq2DkKoFAQEYmsbNYLOhgeKBRERCJr/8BwQYe4AIWCiEhkFfLazCMUCiIiEdU+ejazdjSLiEx7bQU+mxkUCiIikTUyQqr2KYiICO29KSriRmN1RcHWqVAQEYmo9p4Us2oricUKM8QFKBRERCKrrTdVsMtwjlAoiIhEVKHHPQKFgohIZLX3FPZsZlAoiIhEkruzr08tBRERAboGhhnOeEFPXAOFgohIJI1ehrOAQ1yAQkFEJJLaenInrhXybGZQKIiIRFIxBsMDhYKISCS1jQ6Gp1AQEZn22ntTxGNGUwGHuACFgohIJLX3pphVmyzoEBegUBARiaRCX4ZzhEJBRCSCdnUOcFRTVcHXq1AQEYkYd2dHZz8LZ9QUfN2hhoKZvWhmT5rZejNbO87jZmbXmtkWM9toZivCrEdEpBR09A3RP5Rh0czCh0KiAOt4p7u3T/DY2cCxwe0twL8HP0VEpq0dnQMALJpRXfB1F7v76Hzge57zCNBkZvOLXJOISFHt6OgHYPGsMus+Ahy418zWmdnl4zy+ANiRd39nME9EZNra0ZkLhUVF2KcQdvfR6e6+28zmAPeZ2bPu/kDe4+MdgOtjZwSBcjnA4sWLw6lURCQidnQMMLM2SW1lIXr4Xy3UloK77w5+tgJ3AKeOWWQnsCjv/kJg9zivs9rdW9y9pbm5OaxyRUQiYWdnf1H2J0CIoWBmtWZWPzINnAVsGrPYncCfBEchnQZ0ufuesGoSESkFOzr6WViEI48g3O6jucAdZjaynlvc/RdmdgWAu18P/Ax4P7AF6Ac+GmI9IiKRl8k6u/YP8L4Ti3PMTWih4O5bgTeNM//6vGkH/jSsGkRESs3e7kGGM86imWXWfSQiIodu5HDUYhx5BAoFEZFIGT1xrUj7FBQKIiIRsqOjHzOKMhgeKBRERCJlR0c/8xqqqEzEi7J+hYKISITs6Owv2v4EUCiIiETKjo6Bou1PAIWCiEhkpNIZ9vYMFu1wVFAoiIhExq7OAdyLdzgqKBRERCKj2IejgkJBRCQyRk9cU/eRiIjs6OwnGY8xt7445yiAQkFEJDJ2dgywYEY1sdh4l5opDIWCiEhE7OjsZ2GRrqMwQqEgIhIROzr6i7qTGRQKIiKR0JtK09k/XNTDUUGhICISCVE48ggUCiIikbA9CIXF6j4SEZFiX1xnhEJBRCQCtrX30VCVoKmmoqh1KBRERCJgS2svy+fUYVa8cxRAoSAiEgkvtOVCodgUCiIiRba/f4j23iGFgoiI5FoJgEJBRERy+xMAljfXF7kShYKISNFtae0lmYixoMjjHoFCQUSk6La09rJ0di3xIo6OOkKhICJSZFsicuQRFCAUzCxuZk+Y2d3jPHapmbWZ2frg9vGw6xERiZLB4Qw7OwciEwqJAqzjs8AzQMMEj//Q3T9VgDpERCJna1sf7tE48ghCbimY2ULgHOCGMNcjIlKqtkTocFQIv/vom8AXgOwBlvmgmW00s9vNbFHI9YiIRMqW1l5iBktm1Ra7FCDEUDCzc4FWd193gMXuApa4+xuBXwI3TfBal5vZWjNb29bWFkK1IiLF8UJrL4tm1lBVES92KUC4LYXTgfPM7EXgVuBdZnZz/gLuvs/dU8HdbwMrx3shd1/t7i3u3tLc3BxiySIihfVCWy/Lm6PRdQQhhoK7X+XuC919CXARcL+7X5y/jJnNz7t7Hrkd0iIi00Im62xt74vM/gQozNFHr2Jm1wBr3f1O4DNmdh6QBjqASwtdj4hIsezo6GconWXZdAsFd18DrAmmr86bfxVwVSFqEBGJmtExjyIUCjqjWUSkSEYOR11WavsUzGyZmVUG02ea2WfMrCnc0kREytuW1l6a6ytprC7uJTjzTbal8GMgY2bLge8AxwC3hFaViMg0ELUjj2DyoZB19zRwAfBNd/8cMP8gzxERkQm4++h1maNksqEwbGYfAS4BRga2i057R0SkxLT1pOgZTJdsKHwUWAX8nbtvM7NjgJsP8hwREZnAU3u6ATh2brRCYVKHpLr708BnAMxsBlDv7v8QZmEiIuXs15vbqEzEWLF4RrFLeZXJHn20xswazGwmsAG40cz+OdzSRETK16+fa2PVslmRGfNoxGS7jxrdvRv4A+BGd18JvCe8skREytdL+/rY1t7HmcdFbyy3yYZCIhin6A95ZUeziIgchjWbc6M9n3n8nCJX8lqTDYVrgHuAF9z9d2a2FHg+vLJERMrXms2tLJlVw5LZ0biGQr7J7mi+Dbgt7/5W4INhFSUiUq4GhzM8vHUfF715cbFLGddkdzQvNLM7zKzVzPaa2Y+DS22KiMgheHRbB4PDWd5xfPT2J8Dku49uBO4EjgIWkLti2o1hFSUiUq7WbG6lMhFj1dJZxS5lXJMNhWZ3v9Hd08HtP4BoxpyISIT9enMbpy2N3qGoIyYbCu1mdrGZxYPbxcC+MAsTESk32/f1s7W9jzMj2nUEkw+Fj5E7HPVlYA9wIbmhL0REZJLWPNcKRPNQ1BGTCgV33+7u57l7s7vPcfcPkDuRTUREJunXm9s4elYNx0TwUNQRR3LltT+fsipERMpc9+Awv31hXyTPYs53JKFgU1aFiEiZu/Wx7QwMZ/hQy6Jil3JARxIKPmVViIiUseFMlhsfepFVS2dx4oLGYpdzQAc8o9nMehj/w9+A6lAqEhEpM/+9cQ97ugb5+wtOKnYpB3XAUHD3+kIVIiJSjtyd1Q9sZfmcOt4R8f0JcGTdRyIichC/fWEfT+/p5hNvO4ZYLPq7YhUKIiIh+vaDW5ldl+T8kxcUu5RJUSiIiITkub09rNncxiWrlkR2WIuxFAoiIiG54cGtVFXEuPi0o4tdyqSFHgrBWElPmNlrrthmZpVm9kMz22Jmj5rZkrDrEREphNaeQX76xG4uXLmQGbXJYpczaYVoKXwWeGaCxy4DOt19OfAN4KsFqEdEJHQ3P/wSw9ksHzv9mGKXckhCDYXgQjznADdMsMj5wE3B9O3Au80s+rvnRUQOYHA4w/cfeYl3v24uS5vril3OIQm7pfBN4AtAdoLHFwA7ANw9DXQB0bzyhIjIJP3k8V109g/z8beVVisBQgwFMzsXaHX3dQdabJx5rzmD2swuN7O1Zra2ra1tymoUEZlq2azznd9s5cQFDbzlmJnFLueQhdlSOB04z8xeBG4F3mVmN49ZZiewCMDMEkAj0DH2hdx9tbu3uHtLc3P0zwgUkenr18+18UJbHx8/Yyml2BseWii4+1XuvtDdlwAXAfe7+8VjFrsTuCSYvjBYRgPtiUjJuuE3W5nXUMX7T5pf7FIOS8HPUzCza8zsvODud4BZZraF3PUZ/qLQ9YiITJWnd3fz0JZ9XPLWJSQTpXka2AEHxJsq7r4GWBNMX503fxD4UCFqEBEJ2w0PbqUmGeePTl1c7FIOW2lGmYhIxOzeP8CdG3bz4TcvorGmotjlHDaFgojIFLjxoW04cNkZpXcYaj6FgojIEeoeHOY/H9vBOSfNZ+GMmmKXc0QUCiIiR+iWR7fTm0pz+duXFruUI6ZQEBE5AkPpLDc+tI3Tl0f/+suToVAQETkC/7V+F3u7U1z+9mXFLmVKKBRERA6Tu/PtB7fyunn1vP3Y2cUuZ0ooFEREDtNP1+/iub29XP720hzSYjwKBRGRw7Cjo5+/+ulTtBw9g/PedFSxy5kyCgURkUOUzmT57K1PYMA3PnwyiXj5fJQWZJgLEZFycu39W3h8+36u/cgpLJpZ2ucljFU+8SYiUgCPbevguvuf5w9WLCirbqMRCgURkUna3z/E5364noUzarjm/BOLXU4o1H0kIjIJ2azzuR+up7VnkNuueCt1leX58amWgojIJPzbmi38z+Y2rj73BE5e1FTsckKjUBAROYiHtrTzz/c9x/knH8XFpx1d7HJCpVAQETmAl7sG+cx/PsGy5jq+8gcnlc1JahNRKIiITGA4k+VPb3mcweEM/37xSmqS5bkfIV/5v0MRkcP0lZ89y7qXOvnXj5zC8jl1xS6nINRSEBEZx90bd/Pdh7bx0dOX8PtleD7CRBQKIiJjbGnt5f/cvpEVi5u46uzXF7ucglIoiIjk6Uul+eTN66iqiPOtP15BMjG9Pia1T0FEJJDNOl+4fSMvtPXy/cvewvzG6mKXVHDTKwJFRA7gm796nv9+cg9/cfbrOH15eVw051ApFEREyF1W89pfPc8ftizkE29bWuxyikahICLT3hPbO/n87Rs59ZiZ/O0Hyv8EtQNRKIjItLana4BPfG8d8xqquP7ildNux/JYob17M6sys8fMbIOZPWVmfz3OMpeaWZuZrQ9uHw+rHhGRsVLpDFfcnDtj+TuXtDCzNlnskoouzKOPUsC73L3XzCqA35jZz939kTHL/dDdPxViHSIi47rmrqfZsGM/11+8gmPn1he7nEgILRTc3YHe4G5FcPOw1icicihuW7uDHzy6nSvesYz3nTi/2OVERqidZ2YWN7P1QCtwn7s/Os5iHzSzjWZ2u5ktmuB1LjeztWa2tq2tLcySRWQa2LSri7/86SbeumwWV551XLHLiZRQQ8HdM+5+MrAQONXMxl6/7i5gibu/EfglcNMEr7Pa3VvcvaW5uTnMkkWkzHUNDHPFzeuYWZvk2o+cQiI+vXcsj1WQreHu+4E1wPvGzN/n7qng7reBlYWoR0SmJ3fnL3+6iT1dg1z3RyuYXVdZ7JIiJ8yjj5rNrCmYrgbeAzw7Zpn8jrzzgGfCqkdE5CeP7+KuDbv53HuOZeXRM4pdTiSFefTRfOAmM4uTC58fufvdZnYNsNbd7wQ+Y2bnAWmgA7g0xHpEZBp7sb2Pq/9rE6ceM5NPnrm82OVEVphHH20EThln/tV501cBV4VVg4gI5K6g9tkfriceM77x4ZOJx6bvGcsHo1FSRaTs/csvn2fDjv18649WsKBp+o18eii0211Eytrj2zv5tzVbuHDlQs55o85HOBiFgoiUrcHhDFfetoF5DVVc/fsnFLuckqDuIxEpW/94z2a2tvVx82VvoaGqotjllAS1FESkLD22rYPvPLSNi09bzBnHTs8L5hwOhYKIlJ3+oTSfv30DC2dUc9XZry92OSVF3UciUna+9ovNvLSvn1svP43aSn3MHQq1FESkrDy+vZObHn6RP1l1NKctnVXsckqOQkFEysZQOstVP36SeQ1VfP69xxe7nJKkdpWIlI3VD7zA5r093PAnLdTraKPDopaCiJSFF9p6ufb+LZxz0nzec8LcYpdTshQKIlLyslnnqp88SVUixpfO00lqR0KhICIl7/Z1O3lsWwdfPOf1zKmvKnY5JU2hICIlrbNviK/8/BlOXTKTP2wZ94q+cggUCiJS0r52z2a6B9Nc84E3YKYhsY+UQkFEStYT2zu59Xfb+ehbl/C6eQ3FLqcsKBREpCRlss5f/dcm5tRX8me/d1yxyykbCgURKUk/ePQlNu3q5i/POYE6DWUxZRQKIlJy2ntTfP2ezZyxfDbn6sI5U0qhICIl52u/eJaBoQxfPk87l6eaQkFESsoT2zv50dqdXHbGMSyfU1fscsqOQkFESkY263zpzqeYU1/Jp999bLHLKUsKBREpGT9au4ONO7v44jmv187lkCgURKQkdPUP87V7NnPqkpmc96ajil1O2VIoiEjkDWeyXHP30+zvH9LO5ZCp/SUikfb49k7+70+e5NmXe/jkmcs44SiduRym0ELBzKqAB4DKYD23u/uXxixTCXwPWAnsAz7s7i+GVZOIlI7uwWH+6Z7NfO+Rl5hbX8Xq/7WSs94wr9hllb0wWwop4F3u3mtmFcBvzOzn7v5I3jKXAZ3uvtzMLgK+Cnw4xJpEJOL2dg/y3Ye2ccsj2+kdSnPJqiVc+d7jtWO5QELbyu7uQG9wtyK4+ZjFzge+HEzfDlxnZhY8V0Smkad3d/Mfv93GHU/sIpN1zj5pPp98xzJOXNBY7NKmlVCj18ziwDpgOfAtd390zCILgB0A7p42sy5gFtAeZl0iEg2Dwxl+9uQebn7kJR7fvp+qihgfOXUxHz9jKYtn1RS7vGkp1FBw9wxwspk1AXeY2YnuvilvkfEOIXhNK8HMLgcuB1i8eHEotYpI+Nyd7R39PPh8Ow9tyd26B9MsnV3LX517AheuWEhjTUWxy5zWCtJJ5+77zWwN8D4gPxR2AouAnWaWABqBjnGevxpYDdDS0qKuJZES0do9yIadXWza1cVTu7vZtKuLl7sHATiqsYr3vmEeF5yygFXLZukw04gI8+ijZmA4CIRq4D3kdiTnuxO4BHgYuBC4v1T3JwwOZ2jrSdHak2Jfb4r6qgoWNFUzr7GKZEKng0RFNusMZbIMZbKkM04m62Q999MMYmaMfDb1pzL0ptL0ptJk3Wmuq2ROfRUN1Ql9gOUZ+dvf1zdER1+KnZ0DPP5SJ+u2d7KjYwAAM1jWXMdpS2ey4ugZnLF8NsfMrtV2jKAwWwrzgZuC/Qox4EfufreZXQOsdfc7ge8A3zezLeRaCBeFWM9h6RkcZmtbHy/u66Ozb4jO/mH29w/R0T9MW88gbT0p2npSdA+mx32+GcysSWKWuyhIJus4uQ+fWPAhVFeVYG59FXMbq5jXUElFPEbGnUzGSWed7oFhuoLbcNZZ0FTFohk1LJxZw/yGKmbWJZldW8msuiS1kzxCYziTZSidpSYZL8g/pruTSmfpHhymdzBN/1CGvlTuZ1tPit1dA+zZP8jenkGG0lmy7mSz4DgxMxJxIx6LETewYNtZ3jaMxQwDhtJZBtNZBocyDKZzH+r9qdy6BoYzpLNH/p0jmYgxp76Soxqrmd9UxbzGKmbUJKmrTFBflaA2mcAMsg5Zd9whZhCPGTEzkokY1ck4Nck4tckEM2sn/3srluFMlo6+Idp6UrT3pnhubw+bduW++W/b18fYr3LN9ZW0HD2DS1Yt4ZTFTbx+fgM1yWi/R8mxUvti3tLS4mvXrg3t9XftH+C+p17m/s1tbH65m73dqdcs01CVYEZtkjn1lTTXV9Jcl/s5p76K5oZKZtUm6RlMs2v/ALv3D9Dak3uNuBnx2CsfwCPfULsH0+ztyn0gvtw1SCbrxGJGIpZbvqGqgqaaChqrK4jHjF2dA+zsHGAok31NbXWVCY5qquKopmrmN1ZRV5mguiJOdTJBOpPl+dZentvbwwttvQxnnETMaKyuoKG6AgNS6ZFv0VkS8RiViRjJRIzaZILFM2s4elbuNqe+ikTcSMRiJOLGwFCGzv4h9vcP09k/xN7uFC93DbCna5D23hTdA+lx683XXF/JvIYqqipimBnxIKwywXZ6JVRzH7SZ4AN+5Nu+e+4Du7IiTlXwwVtbmaA2+FmTjJOMx6lIGMl4bHT7xmKvrCvrufUB1Cbj1FUmcodCGqNfANp6UuztHmR31yB7ugZ4uWuQ4cyR/R/VJuPMbaiiub6S2cHf1KzaJLPrcz9n1VUyuy7JnPoqqpPxI1pXvmzW2dM9yEvtfbT3DdHVn/vi09E3xO79ud/fnq4B2nuHXvPcoxqrOHFBI284qpH5TVXMqk0yszbJ3IYq5jdWqRUQMWa2zt1bDrbctIvu/qE0W1p7eW5vL3v2D5BKZ0mlMwwMZ3hi+36e2t0NwLLmWs5Y3syyObUsa67jmNm1zKpN0lhdQSJe/O6gbNZpDT6cOvqGaO/NNd9f7hpk9/4BdncNsGlXF32p3HsbsaCpmuPn1fOO45uZUZMcbYF0DQwDUBnPhUBFPEY6m82FRDpL92Cap/d0c89TL0/q2/bsutyHw8IZ1ZyyeEYQPAnqqyqoDz6gaysTVCfjNNdVMrehdLvZ3J2B4Qy9g2m6B9P0pdKv6Ypyf+VLwFA6S/9wJteCGUqzr3eI1p5BWntStHWneGZ3Nw/0puiZoPU5uy7Jghk1LGyqxozRVtfAcAYj14KKB18q6quCbV6VIB4zBoczDAxl6B/KsLNzgG3tfa/6+xhRX5lgXmPuy8WJCxqY21DF7CCYZtdV5v4f6ipD3KpSLNMmFP7n2Va+dOdT7Ojsf1VTN2ZQVREnmYixvLmOq85+Hb93wlyWNkd7nPZYzJjXmOu6OBh3Z3A49y39SL9lpjNZdu8fZF9finTWGQ765quTcWbUVNBUkwvOiggEZ6GYGTXJBDXJBHOmcASGweFMrp++d4j2vhTtwT6rnZ397Owc4Jk93WBQm8yF7MzaJPBKy2k47ezaP0jPYA+9qTSZ4PdUnYxTlYgzv6mKVctmjX7paa5P0lid+/2VakDLkZs2oTCrLslJCxr54IqFHDe3juPm1bNoRs20+OM3synrckjEYyyeVaNjyAugqiLOgqZqFjRVF7sUmUamTSi8cWET3/rjFcUuQ0Qk0sr/a7KIiEyaQkFEREYpFEREZJRCQURERikURERklEJBRERGKRRERGSUQkFEREaV3IB4wdXZnh/noUag6wDzxj4+cn+8ZWZzeFd/G6+GyTw+XWo/2LRqn/zjqj0atR/o/njT+fMKXfux7n7wa5u6e0ndgNWTnZ8/b+zjI/fHW4bc0N5TVptqn9y0alftpVb7ge5PUG/+vKJv9/Fupdh9dNchzL/rAI/fNYllDtXBnj/da5/M9OFQ7a+dp9oPbKpqP9D98aaPtO7JvMahbPfXKLnuo0Iws7U+iXHHo0i1F4dqLw7VPvVKsaVQCKuLXcARUO3FodqLQ7VPMbUURERklFoKIiIyquxDwcy+a2atZrbpMJ670syeNLMtZnat5V101sw+bWabzewpM/va1FY9uo4pr93Mvmxmu8xsfXB7/9RXHt52Dx6/0szczGZPXcWvev0wtvvfmNnGYJvfa2ZHlUjdXzezZ4Pa7zCzpqmuO1hPGLV/KPj/zJrZlPfdH0nNE7zeJWb2fHC7JG/+Af8fptzhHBJVSjfg7cAKYNNhPPcxYBVgwM+Bs4P57wR+CVQG9+eUUO1fBq4sxe0ePLYIuAd4CZhdKrUDDXnLfAa4vkTqPgtIBNNfBb5aQtv89cDxwBqgJSo1B/UsGTNvJrA1+DkjmJ5xoPcX1q3sWwru/gDQkT/PzJaZ2S/MbJ2ZPWhmrxv7PDObT+4f+WHP/Wa+B3wgePiTwD+4eypYR2sJ1V4QIdb+DeALQGg7w8Ko3d278xatDaP+kOq+193TwaKPAAunuu4Qa3/G3TeHUe+R1DyB9wL3uXuHu3cC9wHvK8b/ctmHwgRWA59295XAlcC/jbPMAmBn3v2dwTyA44C3mdmjZvZrM3tzqNW+2pHWDvCpoDvgu2Y2I7xSX+OIajez84Bd7r4h7ELHccTb3cz+zsx2AH8MXB1irfmm4u9lxMfIfVMtlKmsvVAmU/N4FgA78u6PvI+Cv79pc43mEWZWB7wVuC2va65yvEXHmTfy7S5Brol3GvBm4EdmtjRI8tBMUe3/DvxNcP9vgH8i988eqiOt3cxqgC+S684oqCna7rj7F4EvmtlVwKeAL01xqa8uZorqDl7ri0Aa+MFU1jiRqay9UA5Us5l9FPhsMG858DMzGwK2ufsFTPw+Cv7+pl0okGsd7Xf3k/NnmlkcWBfcvZPch2d+U3khsDuY3gn8JAiBx8wsS24ck7YwC2cKanf3vXnP+zZwd5gF5znS2pcBxwAbgn+4hcDjZnaqu78c8drHugX4b0IOBaao7mCn57nAu8P+4pNnqrd5IYxbM4C73wjcCGBma4BL3f3FvEV2Amfm3V9Ibt/DTgr9/sLcYRGVG7CEvJ1BwG+BDwXTBrxpguf9jlxrYGQHz/uD+VcA1wTTx5Fr9lmJ1D4/b5nPAbeWynYfs8yLhLSjOaTtfmzeMp8Gbi+Rut8HPA00h7Wtw/57IaQdzYdbMxPvaN5GrgdiRjA9czKmsp26AAADrklEQVTvb8rfU9i/6GLfgP8E9gDD5FL3MnLfOH8BbAj+4K+e4LktwCbgBeA6XjnZLwncHDz2OPCuEqr9+8CTwEZy37Tml0rtY5Z5kfCOPgpju/84mL+R3Bg0C0qk7i3kvvSsD25TftRUiLVfELxWCtgL3BOFmhknFIL5Hwu29xbgo4fy/zCVN53RLCIio6br0UciIjIOhYKIiIxSKIiIyCiFgoiIjFIoiIjIKIWClAUz6y3w+m4wsxOm6LUylhs9dZOZ3XWwkUjNrMnM/vdUrFtkLB2SKmXBzHrdvW4KXy/hrwwEF6r82s3sJuA5d/+7Ayy/BLjb3U8sRH0yvailIGXLzJrN7Mdm9rvgdnow/1Qz+62ZPRH8PD6Yf6mZ3WZmdwH3mtmZZrbGzG633DUFfjAyln0wvyWY7g0Gu9tgZo+Y2dxg/rLg/u/M7JpJtmYe5pUBAOvM7Fdm9rjlxtM/P1jmH4BlQevi68Gynw/Ws9HM/noKN6NMMwoFKWf/AnzD3d8MfBC4IZj/LPB2dz+F3Gilf5/3nFXAJe7+ruD+KcCfAScAS4HTx1lPLfCIu78JeAD4RN76/yVY/0HHqwnG9Xk3uTPNAQaBC9x9BblrePxTEEp/Abzg7ie7++fN7CzgWOBU4GRgpZm9/WDrExnPdBwQT6aP9wAn5I1Y2WBm9UAjcJOZHUtuxMmKvOfc5+75Y+Q/5u47AcxsPbmxbn4zZj1DvDKw4Drg94LpVbwy9v0twD9OUGd13muvIzeWPuTGuvn74AM+S64FMXec558V3J4I7teRC4kHJlifyIQUClLOYsAqdx/In2lm/wr8j7tfEPTPr8l7uG/Ma6TypjOM/z8z7K/snJtomQMZcPeTzayRXLj8KXAtuesuNAMr3X3YzF4EqsZ5vgFfcff/d4jrFXkNdR9JObuX3HULADCzkSGNG4FdwfSlIa7/EXLdVgAXHWxhd+8id6nOK82sglydrUEgvBM4Oli0B6jPe+o9wMeC8fwxswVmNmeK3oNMMwoFKRc1ZrYz7/bn5D5gW4Kdr0+TG/Ic4GvAV8zsISAeYk1/Bvy5mT0GzAe6DvYEd3+C3AibF5G7oE2Lma0l12p4NlhmH/BQcAjr1939XnLdUw+b2ZPA7bw6NEQmTYekioQkuFrcgLu7mV0EfMTdzz/Y80SKSfsURMKzErguOGJoPwW47KnIkVJLQURERmmfgoiIjFIoiIjIKIWCiIiMUiiIiMgohYKIiIxSKIiIyKj/Dxh2srbdC5LFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.lr_find()\n",
    "learn.recorder.plot(skip_end=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 36:59 <p><table style='width:375px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "    <th>time</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>0</th>\n",
       "    <th>2.588984</th>\n",
       "    <th>2.512885</th>\n",
       "    <th>0.536470</th>\n",
       "    <th>12:16</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>2.433887</th>\n",
       "    <th>2.368720</th>\n",
       "    <th>0.558128</th>\n",
       "    <th>12:21</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>2.323759</th>\n",
       "    <th>2.332253</th>\n",
       "    <th>0.564441</th>\n",
       "    <th>12:21</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4VFX6wPHvm94TIKEGCFU6IQQEQUBREVCxoGLvrHVFdP2hq2tZC9bFulixoa4VC0UFUUTpvfdOgNBCICH1/P64N5OZzKQAmUySeT/PM09uOffOmWGYd04XYwxKKaUUQICvM6CUUqr60KCglFLKQYOCUkopBw0KSimlHDQoKKWUctCgoJRSykGDglJKKQcNCkoppRw0KCillHII8nUGTlR8fLxJSkrydTaUUqpGWbRo0X5jTEJ56WpcUEhKSmLhwoW+zoZSStUoIrKtIum0+kgppZSDBgWllFIOGhSUUko51Lg2BaVU7ZGXl8fOnTs5fvy4r7NSa4SFhZGYmEhwcPBJXa9BQSnlMzt37iQ6OpqkpCRExNfZqfGMMRw4cICdO3fSokWLk7qHVh8ppXzm+PHj1KtXTwNCJRER6tWrd0olLw0KSimf0oBQuU71/fSboJCbX8gXC3egy48qpVTpvBYURCRMROaLyDIRWSUiT3hIEyoi/xORjSIyT0SSvJWfl39Zz4NfLefn1Xu99RRKqRrmwIEDJCcnk5ycTMOGDWnSpIljPzc3t0L3uOmmm1i3bp2Xc1p1vNnQnAOcbYw5KiLBwGwRmWqMmeuU5hbgkDGmtYiMAJ4DrvRGZmLDrZb4kCC/KRwppcpRr149li5dCsDjjz9OVFQUDzzwgEsaYwzGGAICPH93TJgwwev5rEpe+4Y0lqP2brD9KFl3Mwz40N7+ChgoXqpgbNcwGoC48JPrpqWU8h8bN26kU6dO3H777aSkpJCWlsbIkSNJTU2lY8eOPPnkk460ffv2ZenSpeTn5xMXF8eYMWPo2rUrvXv3Zt++fT58FSfHq11SRSQQWAS0Bt4wxswrkaQJsAPAGJMvIhlAPWB/5WfG+qMtCkpVT0/8sIrVu49U6j07NI7hsQs7ntS1q1evZsKECYwfPx6AsWPHUrduXfLz8znrrLMYPnw4HTp0cLkmIyOD/v37M3bsWEaPHs3777/PmDFjTvl1VCWv1qUYYwqMMclAItBTRDqVSOKpVOD2vS0iI0VkoYgsTE9PP6m8BNgFEG1nVkpVRKtWrejRo4dj/7PPPiMlJYWUlBTWrFnD6tWr3a4JDw9n8ODBAHTv3p2tW7dWVXYrTZUMXjPGHBaR34DzgZVOp3YCTYGdIhIExAIHPVz/NvA2QGpq6kl9rRdFH+19pFT1dLK/6L0lMjLSsb1hwwZeeeUV5s+fT1xcHNdee63HsQAhISGO7cDAQPLz86skr5XJm72PEkQkzt4OB84B1pZI9j1wg709HPjVeOlbW7T6SCl1ko4cOUJ0dDQxMTGkpaXx008/+TpLXuPNkkIj4EO7XSEA+MIY86OIPAksNMZ8D7wHfCwiG7FKCCO8lRlBq4+UUicnJSWFDh060KlTJ1q2bEmfPn18nSWvkZpWnZKammpOZpGdvzbu5+p35/G/kb04vWU9L+RMKXWi1qxZQ/v27X2djVrH0/sqIouMManlXes/nfbt6qPCmhUDlVKqSvlNUHBUH2mrglJKlcp/goKj+5FPs6GUUtWa3wSFonEKBTWsDUUppaqS3wSFopLCw9+u8G1GlFKqGvOfoGD/3XEwm+0Hssg8nufT/CilVHXkN0HBudKo3wszGfb6nz7Li1KqehgwYIDbQLRx48Zx5513lnpNVFQUALt372b48OGl3re8rvPjxo0jKyvLsT9kyBAOHz5c0ax7jd8EhZIlg837j/koJ0qp6uKqq67i888/dzn2+eefc9VVV5V7bePGjfnqq69O+rlLBoUpU6YQFxd30verLH4TFGatr/yJV5VSNdvw4cP58ccfycnJAWDr1q3s3r2b5ORkBg4cSEpKCp07d+a7775zu3br1q106mTN8Zmdnc2IESPo0qULV155JdnZ2Y50d9xxh2PK7cceewyAV199ld27d3PWWWdx1llnAZCUlMT+/db31Msvv0ynTp3o1KkT48aNczxf+/btue222+jYsSPnnXeey/NUliqZEK86qGkjt5XyO1PHwJ5K7gjSsDMMHlvq6Xr16tGzZ0+mTZvGsGHD+Pzzz7nyyisJDw/n22+/JSYmhv3799OrVy8uuuiiUtc//u9//0tERATLly9n+fLlpKSkOM49/fTT1K1bl4KCAgYOHMjy5cv5+9//zssvv8zMmTOJj493udeiRYuYMGEC8+bNwxjD6aefTv/+/alTpw4bNmzgs88+45133uGKK67g66+/5tprr62c98rmNyUFTyOZ/9qkpQel/J1zFVJR1ZExhocffpguXbpwzjnnsGvXLvbuLX0p31mzZjm+nLt06UKXLl0c57744gtSUlLo1q0bq1at8jjltrPZs2dzySWXEBkZSVRUFJdeeil//PEHAC1atCA5ORnw3tTcflNSaBgb5nbs6nfmsXXsUB/kRinlpoxf9N508cUXM3r0aBYvXkx2djYpKSl88MEHpKens2jRIoKDg0lKSvI4VbYzT6WILVu28OKLL7JgwQLq1KnDjTfeWO59yqrVCA0NdWwHBgZ6pfrIb0oKA05L8Hh835Gy/4GUUrVbVFQUAwYM4Oabb3Y0MGdkZFC/fn2Cg4OZOXMm27ZtK/Me/fr1Y+LEiQCsXLmS5cuXA9aU25GRkcTGxrJ3716mTp3quCY6OprMzEyP95o0aRJZWVkcO3aMb7/9ljPPPLOyXm65/CYolKbnMzN8nQWllI9dddVVLFu2jBEjrNn7r7nmGhYuXEhqaioTJ06kXbt2ZV5/xx13cPToUbp06cLzzz9Pz549AejatSvdunWjY8eO3HzzzS5Tbo8cOZLBgwc7GpqLpKSkcOONN9KzZ09OP/10br31Vrp161bJr7h0fjN19qrdGQx9dbbb8VYJkcy4fwAAj3+/ikEdG9K7lU6trVRV0KmzvUOnzq6AkMDil9q9eR2GdmkEwNnt6gOQkZXHB39t5ap35vokf0opVR34TUNz6/pRPDK0PRd1bUz9GKvRef6W6RzNKQDgk3ll1xkqpZQ/8JuSgohw65ktHQEBIDo0iKM51sLaQQHFPQfu/XwJM9fuq/I8KuWPaloVdnV3qu+n3wQFTyJDgzhqT38RGlT8Vny3dDc3fbDAV9lSym+EhYVx4MABDQyVxBjDgQMHCAtz74JfUV6rPhKRpsBHQEOgEHjbGPNKiTR1gPeBVsBx4GZjzEpv5amkqNAgjtnVRwez3GdNzc0vJCTIr+OmUl6VmJjIzp07SU9P93VWao2wsDASExNP+npvtinkA/cbYxaLSDSwSER+McY4D+d7GFhqjLlERNoBbwADvZgnF5GhgUxfY1UTvT97i9v5V2ds4IFBp1VVdpTyO8HBwbRo0cLX2VBOvPYz2BiTZoxZbG9nAmuAJiWSdQBm2GnWAkki0sBbeSqpKCA8M2WNo23B2Zu/bdRirVLKr1RJ3YiIJAHdgHklTi0DLrXT9ASaA27lHhEZKSILRWRhZRYzz2xjTUT19qzNHs8XmuLAoZRS/sDrQUFEooCvgVHGmCMlTo8F6ojIUuAeYAlWtZMLY8zbxphUY0xqQoLn6SpOxmMXdnTZv6WvezF216Est2NKKVVbeTUoiEgwVkCYaIz5puR5Y8wRY8xNxphk4HogAXCv3PeS+jGhLvufzHUfqzBl5Z6qyo5SSvmc14KCWFMGvgesMca8XEqaOBEJsXdvBWZ5KE14TUxYMNf3bu7Y/+jmniz45zksf/w8x7H5Ww5WVXaUUsrnvDb3kYj0Bf4AVmB1SQWrt1EzAGPMeBHpjdVttQBYDdxijDlU1n1Pdu6j8hQWGgKcBrCt35vJef+ZBcDUe8+kTf0oCg3aRVUpVSNVdO4jr3VJNcbMBjwvU1ScZg7Qxlt5OBHOAQGgbYNox/bgV/4gITqU9MwcXX9BKVWr6c/eMvRvW9yonZ5preF6zEPXVaWUqi00KJThjWtS3I51fOwnksZM5pFJK0gaM5k3Zm70Qc6UUso7NCiUISo0iPdu8FwF98nc7QC88NM6ksZM5tmpa6oya0op5RUaFMpR0QV33vrd8wA4pZSqSTQolCMixG+WnFBKKQ0KFbHyiUFseXZIuemycrURWilVs2lQqICo0CBEhA6NYspM1+FfP1VRjpRSyju0buQEfHVHb47m5PPOrM10b16Hs9s1oO0jU32dLaWUqjReG9HsLd4a0Xyylu44TLO6EaT8+xfHMR3gppSqbnw+otlfJDeN83UWlFKq0mibghfUtNKXUkoV0aBQSa5ILV4b6K1Zm8nJL/BhbpRS6uRoUKgkYy/tUrw9dS3PTF6DMYaM7Dwf5koppU6MBoVKEhAg/HB3X8f+il0ZvPvHFro+8TM/rdKFepRSNYMGhUoUFxHs2A4MENbuyQTgbx8v8lWWlFLqhGhQqEQJ0cXLey7Yeojm9SIc+4WF2vislKr+NChUorDgQB4e0s6xP3PdPsd2y4enkJNfQOZxbWNQSlVfGhQq2ch+rRzbS7Yfdjl39ou/0/nxn6s6S0opVWFeCwoi0lREZorIGhFZJSL3ekgTKyI/iMgyO81N3spPVeqaGOvx+K7D2QBkZGlpQSlVPXmzpJAP3G+MaQ/0Au4SkQ4l0twFrDbGdAUGAC+JSIgX81Qllu3MKPN81ye1tKCUqp68FhSMMWnGmMX2diawBmhSMhkQLSICRAEHsYJJjfb2dd19nQWllDopVdKmICJJQDdgXolTrwPtgd3ACuBeY0yhh+tHishCEVmYnp7u5dyeuvM6NmTuQwN5eEg7Fj5yDpd0KxkLYfuBLB/kTCmlyub1oCAiUcDXwChjzJESpwcBS4HGQDLwuoi4LVpgjHnbGJNqjElNSEjwdpYrRcPYMEb2a0V8VCgDTrPy7DwVxoZ9mb7KmlJKlcqrQUFEgrECwkRjzDcektwEfGMsG4EtQDsP6Wq0YclNWPLouTw/vCsf3NQDgFs+rD7TfyulVBFv9j4S4D1gjTHm5VKSbQcG2ukbAKcBm72VJ1+qE2m1nx/LKZ4oLz0zR2dUVUpVK94sKfQBrgPOFpGl9mOIiNwuIrfbaf4NnCEiK4AZwP8ZY/Z7MU8+N6hjA8d2j6enM1KnwFBKVSNeW2THGDMbkHLS7AbO81YeqqOgwAAeGtyOZ6euBeCX1Xvd0ny1aCdr047wyAUle/AqpZR36YhmH9iXmVPquezcAh74chnvzt5ShTlSSimLBgUfuCK1qWO7UWwYf23cjzGG3PxC2v9rmuPcnozjvsieUsqPaVDwgVYJkY7ttIzjXP3uPFo8NIWsXNdxe2kZ2VWdNaWUn9Og4ANBgQEEB7o3txzLdV3Cc8O+o1WVJaWUAjQo+MyGp4dwpVM1EsCuQ64lgwe/Ws5fm2p1ZyylVDWjQcGHSo5q3nbgmFua39enO8YyPDppJcPe+LNK8qaU8k8aFHxocYn1Fv7x1XIAXr2qm+PYW79v5u7PlrD/aA4fz93Gsh2HdcCbUsprNCj40FMXdwLgk1tOdzm+ancGgzs1dOxPXp5G6lPTHft//3wp72mXVaWUF3ht8Joq37W9mnNlj6YEimuj8+39WlEnMoQ+Y391LMzj7Idlu/lh2W5u6duiqrKqlPITWlLwseDAAAIChDsGFC/jWTRPklJKVTUNCtXEZSnWmgsxYcWFN0+lBGe5+W5LTyil1CnRoFBNNImLID4qhBcu71rha47m1PhF6pRS1YwGhWoiPCSQhY+cy6COxQ3MjWPDAOiRVAeAqFDXJqDM43lVl0GllF/QhuZq7NcHBvD6rxu579y2vDJ9PUO7NObrxTtZvzeT39alcygrj+b1fJ1LpVRtIjWtz3tqaqpZuNC/Vy2bvWE/175nLXe9dexQH+dGKVUTiMgiY0xqeem0+qgGahAT6rK/63A23y3d5aPcKKVqE60+qoHaNIh2bP/7x9WOgWzndmhAYIAQGhToq6wppWo4LSnUcM4jmzs99hMpT/7iw9wopWo6rwUFEWkqIjNFZI2IrBKRez2k+YfT+s0rRaRAROp6K0+1XaEpnn7bGMOkJbvIyS+ejjs9M4ekMZNZvP2Qr7KolKrmvFlSyAfuN8a0B3oBd4mIy6LDxpgXjDHJxphk4CHgd2PMQS/myW/8ti6dUf9bysu/rHccm7ZqDwCXvvmXr7KllKrmvBYUjDFpxpjF9nYmsAZoUsYlVwGfeSs/tc3FyY0BeOKijm7nCgoNWXaJ4a3fNzuON68bAUC7htFu1yilFFRRm4KIJAHdgHmlnI8Azge+ror81AbPD+/K/H8O5Prezd3OZeXmU+DU1bhorecH7am5cwt0egyllGdeDwoiEoX1ZT/KGHOklGQXAn+WVnUkIiNFZKGILExPT/dWVmuUkKAA6keHISKsfGIQ00f35+Y+1qyp2bkFZGQXj3bu9ewMAPYcOe44X5p9R47reg1K+TGvBgURCcYKCBONMd+UkXQEZVQdGWPeNsakGmNSExISKjubNV5UaBCt60fRsXEMAFm5Bew4mOWS5vf1xcE0LeM4k5enAXDThPkkjZnMgaM5rN59hJ7PzOCBL5dXXeaVUtWKN3sfCfAesMYY83IZ6WKB/sB33sqLvygqHbzw8zrenrXZ5dwN78932b/r08UAzFxnBYvuT03n0/nbAPh68U5vZ1UpVU15c/BaH+A6YIWILLWPPQw0AzDGjLePXQL8bIxxX6BYnZD4aGukc1EpoDzjf9/ksv/J3O2VnielVM1SoaAgIq2AncaYHBEZAHQBPjLGHC7tGmPMbEBKO++U7gPgg4rkQ5Wtmd27qKLGTl3r8XhoUOkFyE3pR2lWN4LgQB33qFRtVNH/2V8DBSLSGqtKqAXwqddypU5K/WjXOZFeu6obG54e7HLszDbx5d4np5TFe7bsP8bAl37njZkbTz6TSqlqraJBodAYk49V1TPOGHMf0Mh72VIno3FcOHedVbys54VdGxMcGMCSR88F4JURybx5TQohJX7lh3goGRQUGj6es5WkMZPJzi3gcFYuW/YfBWDhVh0RrVRtVdE2hTwRuQq4Aav7KECwd7KkTsU/BrXjgfNOw2rnt9SJDHGZYnvewwPp9u/iOZLmjDmb7k9Nd7nPrkPZPPrdKgDunLjI0SANEBGiE+4pVVtVtKRwE9AbeNoYs0VEWgCfeC9b6lQ4BwRP6kSG8MeDZxXvR4TQp3U9/ta/JU8Os0ZIn/XSb47zzgEBNCgoVZud8CI7IlIHaGqM8Ulndl1kp/Ks3JVBgAgd7PENANNWpnH7J4vLvVYX91GqZqnURXZE5DcRibFnMF0GTBCRUsceqJqhU5NYl4AAEBFSsRrFkoPjlFK1Q0Wrj2LtKSouBSYYY7oD53gvW8pXckvpeVTSyl0ZXs6JUsoXKhoUgkSkEXAF8KMX86N8rEeLii1nUV67hVKqZqpo76MngZ+wJq1bICItgQ3ey5byldjwYCb/vS9N60YQHRrEM1PW8M4f1upuQ7s0IjY8mE/nbSfzuDWlxgWv/cHKXUfY9MwQAgM0UChV01WopGCM+dIY08UYc4e9v9kYc5l3s6Z8pWPjWGLCghER/u/8do7jb1ydwv8NsvaL5llaucua+PbT+TpFhlK1QUUbmhNF5FsR2Scie0XkaxFJ9HbmlO8FlRjoFh0WRGCAcDgrz+X4o5NWVmW2lFJeUtE2hQnA90BjrNXTfrCPKT/QIj6SQR0bABAQINSNDGF3RjZJYya7pPtjQ+lrXWTnFjBrva6FoVR1V9E2hQRjjHMQ+EBERnkjQ6r6mfnAAJf99Mwcvlm8yy3dxLnbObON+3oXK3dlMOyNPykoNEy6qw/JTeO8lVWl1CmqaElhv4hcKyKB9uNa4IA3M6Zqnmmr9nAsJ9/t+AWvzaag0BokecsHC6o6W0qpE1DRoHAzVnfUPUAaMBxr6gulXEz4c0uZ5w8cy62inCilTkZFex9tN8ZcZIxJMMbUN8ZcjDWQTfmh+Q8PdNl/fngXx7Yx8Pj3q3hthvZYVqomOpWV10YD4yorI6rmqB8TxvDuiQzvnkiXxFjCgwMZ8/VyCg1MXpHG2j2ZABQYw5szN7ldv35vJm0bRLN0x2G6JsbqQDilqpFTWT5L/yf7sRcv70qvlvWICAlCRPj5vv4AZB4vblMYN30DuQXWtBnOA9u+WbyLSUt2cfEbf9LioSkczyuo2swrpUp1KkGhzOlVRaSpiMwUkTUiskpE7i0l3QARWWqn+f0U8qN8qHX9KJrEhbPrcLbH8w8Nbsd3d/UBrLWhR/1vqePcOrtkoZTyvTKrj0QkE89f/gKEl3PvfOB+Y8xiEYkGFonIL8aY1U73jwPeBM43xmwXkfonln1VnUSHlf5xigwNokkdzx+ZotKEUsr3ygwKxpjok72xMSYNq6cSxphMEVmDNfBttVOyq4FvjDHb7XT7Tvb5lO9FhZYdFOpEhHg8d/S4ezdWpZRvnEr1UYWJSBLQDZhX4lRboI69XsMiEbm+KvKjvGPz/mOlnsvOzS91wjztpqpU9XEqvY8qRESigK+BUfaaDCWfvzswEKs6ao6IzDXGrC9xj5HASIBmzZp5O8vqJB10+nIv2b5QVg+jQxoUlKo2vFpSEJFgrIAw0RjzjYckO4Fpxphjxpj9wCyga8lExpi3jTGpxpjUhAT3aRRU9fCvCzo4tj+8uSe3ndmCS7o1AeCiro1d0gYI9G5ZD4DlumCPUtXGCa/RXOEbWz8NPwQOGmM8zpMkIu2B14FBQAgwHxhhjCl1yk1do7n6SsvIpvezvxIgsPlZz2s4F33eRARjDC0emgLoms9KeVtF12j2ZvVRH+A6YIWIFPU/fBhoBmCMGW+MWSMi04DlQCHwblkBQVVvjWLDeWVEMn1bx5eaxrkaSQetKVX9eC0oGGNmU4EBbsaYF4AXvJUPVbWGJTc5ofRDuzRize6STU1KKV+pkt5HSpUmJiyYzfuPcThLG5uVqg40KCifCg8OBGDoq7N9nBOlFGhQUD72y5o9AKVOj6GUqloaFJRPJdWL9HUWlFJONCgonzqtwUnPpKKU8gINCsqn7j2njWNbq5CU8j0NCsqnosOCObudNTlun7G/+jg3SikNCsrnth4onkjvse907KJSvqRBQflcn1bFI6A/nLPNY5pB/5nFw9+uqKosKeW3NCgon3vioo4u+3keFt1ZtzeTT+dtr6osKeW3NCgonwsIEP5+dmvH/uPfrwKsZTqTxkxmyfZDvsqaUn5Hg4KqFvq2KZ4SvV1Dq5vqlwt3AHDJm3+Vel1aRjZHc0595bbs3IJTvodStYEGBVUtOK/fPHfLQQCOHM8r97rez/7KRa+5T5Fx/rhZ3PLBggo996b0o7T/1zS+XbKzgrlVqvbSoKCqhSZx4Xx/dx8AJi9PA2D/0YpNkudpGdC1ezKZsbZiS36vTcsE4L7/LatQeqVqMw0Kqtro2DjWsb1uTya/evhSd64qch7sNm/zgZN+3ojQQMd20pjJFBR6Z+EppWoCDQqq2ggMKF5+Y9C4WR7TPPXjagAysvN48KviX/ZXvj3Xse28muCejOPlPm9+gWsQeGvWpoplWKlaSIOCqlbaNogq8/zaPZnsPpxN1yd+5s+NnksH8+02CYCXfl4HwHPT1pI0ZjLH89wblI/muLZd/LAs7USzrVStoUFB+dbyL2H3ErB/3d9zdpsyk8dFBHPnxMVlpnl39hbH9ml2T6b//mb9+l+247Bb+qPHXXsvrUnTleCU//JaUBCRpiIyU0TWiMgqEbnXQ5oBIpIhIkvtx7+8lR9VDRXkw08PwdsDYPyZMO8tLmgTVuYlv61LZ6mHL3ZnzlVGXy/exVanhugC495ecOS4e5fWilQ7KVUbebOkkA/cb4xpD/QC7hKRDh7S/WGMSbYfT3oxP6q6CQyCuxfC0JcgIBCmPoi8dBrvRrxJ34AVCIXERQSz+NFz+eHuvh5v0Sg2jFv6tiAwQDh4zOqttPdI8Rf64axcBrz4m2P/6nfmsf9ojss9ino7vXh5V8exXs/OqJSXuGFvpiNfStUEXgsKxpg0Y8xiezsTWAOc2KruqvYLj4Met8LffofbZ0PqzfQLXMEnIc/yR+gofuz8J3Xz9tA5Mdbj5cfzCmgSF05BoSHl37+Qk1/AvkzrS79Doxg6NIpxuyb1qemOGVkLCw2r7eqi4d0TuapnU0e6yhjQdu5/ZnF+KY3mSlVHVdKmICJJQDdgnofTvUVkmYhMFZGOHs4rf9GwMwx+jpAH15N14TtENW5H4rJXYFwX+OhiPum1ixBcG4WP5RawMf2oY/+P9fsd23UjQziY5flXelF31gMlfsWv2n3EKU3WKb2c69+fD+AIUkrVBF4PCiISBXwNjDLGlGzBWww0N8Z0BV4DJpVyj5EislBEFqanp3s3w8r3gsOI6H4FcX+bDKOWw4AxcGATfZf+g3mhd/FY0Id8NNRaxjOvoJAuTYpLEbd+tBCwVnSrExnCku3F7Q/f3dXH5Wlu+WCBo+0gNjwYgHFXJjvO5+S7T8x3Imat18+qqnm8GhREJBgrIEw0xnxT8rwx5ogx5qi9PQUIFpF4D+neNsakGmNSExISSp5WtVlcMyso3LuMHUM/ZXZhJ64OnEG/GcOYGv4oX3ZfzYWnua/z3LVpLHUjgksci+O6Xs0d+zPW7uPC160pMt6/sQcALROi+OjmnkBx9dG8zQfI9zBza0XFR4Wc9LVKVTVv9j4S4D1gjTHm5VLSNLTTISI97fyc/NBUVXsFBBDU9mzuyfs7p+e8Qd55z9K+fjipK58i8rUOzG/3Bb0CViNYX97/GNSOuIjiL+Mnh1k1k3UiPX9BN4gJdWyHh1gjnDOP57N0x2GufHsurf85ldwTKDkYl15OUmo6paqbIC/euw9wHbBCRJbaxx4GmgEYY8YDw4E7RCQfyAZGGOOhz6BSQB37S/4w0QSfMQJ63wFpS2HxxySs+JLPQyaf/gsYAAAgAElEQVSxrbA+XxQMIDq3i8s0GcO6Wn0c7hzQildnbHC7d0J0cVCIDLH+W+w6nM1NTpPqfb5gO9f3TuKuTxdzYZdGnN+pEQCjv1hKYaHhcHYe79/Qg4AAcenmuv9oDgWFhryCQsKCi6fUUKo68mbvo9nGGDHGdHHqcjrFGDPeDggYY143xnQ0xnQ1xvQyxpQ+R7Lye2HBgVxzejM+vsWq3kEEGneDC15G7l/HqNw72W3i+UfwF4S+3oX79v2TQQHzCSaf6LAgxz3+7/x2LvcNEAgNKv6ybhxnjZX4bZ3r3Ev//W0TufmFTF6exu2fFA+g+2bxLiYt3c1v69KZu8Uq6G4/4NpI/em8bbR7dBpJYyaz7YD7BH7lycjO48LXZrNuT+YJX6vUidARzapGefqSzpzZxkO7UkgEkwr7clXeI8y5YDrSdzS9ItN4K2Qcq2JHEfDLI7BvLQCDOzV0uXT1k+e77MeEBRMgMH2Na1AY2L4+GdnFvZ88jXxetesIR47nOdoqijz+w2rH9gWvzubFn9ZV7AXb/ty4nxW7MnjhBK9T6kRpUFC1Tu/UHjDwUSIeXANXf0lIyz4wbzy8eTq8ew5J275i7SPFPZFKVukEBAieJko9llNARnZxF9avF+10a4COCA3kuyW7HPtntKoH4DLzamZOPq/P3MhD3yzn+2W7SRozmbnlzPK6xR6VHaDNE8rLvNmmoFSV2jp2qOuBgEBoe571OLYfln0OSz6GH/5O2LQxbOw5jMLk66x5l6T0b9vLuyfy5aKdfLtkF986feHHR4ey81C2S9qVuzL4bP4Ox36zuhH8tcnzF/5n83c40o54ey7jrkzm4m7u4zuNMY4SQslxFSfiwNEc6kWFlp9Q+TUtKSj/EBkPZ9wNd86FW6ZD5+EErf2BkI+GwOs9YPY4yNzr8dKiMQwlBQUIWXa31fHXphAfFeISEABaxLt3ly3NqP8tZV+m+5xLLR6a4thetM11veq3Z22i1zPlT8mxclcG3Z+aziSnoKaUJxoUlH8RgaY94KLX4P51MOxNK2BMfwxebg+fXQVrp9A7qXhAXHBQAL1a1nW71UdztpGdZ/UyCgsOpF6k66/wR4a254Yzkk4oez2fnkHeCYyJeGbKWvYcOc6Og2WPvl6+MwOAifO2eTyvCwupIhoUlP8KjYJu18DN06yJ+c64G3YuhM+v4pPMm3g8/AtaSBo5eYUk1olwu3z7wSwu++8cwPpS3e20EtwfD57FrWe2JCw4kH/bYySS6rnfw5Nx09eXeb6o1/aibcXrRvxvwQ6PaWes2cuOg1lk2+tILNh6yC3NsNdn0+rhKRw4qtNxKA0KSlni28C5T8Lo1TDiUwITU7mBH5gZej9j9tzHQ40W06dZOHcMaMWqJwa5Xd6xcSyZTkuFNq1bHADi7Xr8YclNGH1u23Kz8sbM4pXfPJUaiuZnKgpIAK/P3EhufiFfL9pJYaFhT8ZxksZM5pYPFzLi7bmEBHn+r348r4Bldimi+1PTT6iUomonbWhWyllgMLQbCu2GIpl7YNlnhCz5hHrTRzExJBpyL4X06wGD80jlhrHF60Cc076+yy3PalefJ4d15PLuTQkIgP5tE1i8/RBP2N1U+7dNIDI0kCkr9rhl58BR94blC16bzaZnhrgdf2/2Fp6btpaAALjvf8VLle46nM2Hf2117C/feZguiXGA+0ywbf45lf87vx23ntmC4ED9zeiPNCgoVZrohtD3PugzCrbPgSWfwIovYfGH/BSSyBcF/fm24EwO4jo9d8l2hLDgQK7vXXysa9M4uiTGclqDaIICA0huGkdIUAB5BYW0+edU6jpNxbHniOfFfoa++ofbsaJqpy373dsXNu4rnkn2otf/dPTUOp7vPj34c9PWEhUayHVOeVb+Q38KKFUeEWh+Blz8ptU4feErxMXF8WjwROaG3sX39d+CDb/w8719eGVEsufBdW63FM5oHU/PFnUdVTtFv8wPHsslN7+QgkLDSHvW12HJjVny6LmO69c6jWxObmr96i+a1dXTNB4lFbV/HMvxvGbErsO68py/0qCg1IkIi4HuN5J+5RTOzXmeWXGX0CV/JUwcTtvPzmDYwQlwcEv59ylF39bWJMHbDhzj9V83OtZi+OeQ9tSJDHFbNKhlfGS5y5N68uiklRQWGvaVUhIZ//smj8dV7adBQamT0LFxDLdcMpied4yH0Wvhio+gfgf44yV4NRk+uACWfwF52eXfzMm1vZoB1optPy7f7TgeY4+VKLlgz68PDODCro0rdO+mdcMd2weO5dLy4Slc/a617lXPJPcut8o/aVBQ6iSICCN6NiMmLBiCQqDDMLj2Kxi1Es5+BDJ2wDe3wYunwY+jYfcSa+R0OaLDigfKbXBqBwixq5b6tK7ndk1Rl9eSfrynr8u606nN6zJ9dH8At9LF6PPce0XphMX+SYOCUpUptgn0+wfcswRu+AHaDoKlE+HtATD+TJj3FmQdLPXyuqWs9xBgT3r0wvDiL/kJ9sJApY247tQkluHdE7mlbwvAqppqXT/KcT7eacqLbs3ieGRoe5frj1XCGtWq5tHeR0p5Q0AAtOhnPbJfsHotLfkYpj4IPz8K7S+AbtdBi/5WWlv7Em0GJYUEBbDx6cEEiDgChTjN27TokXPo/tR0l2seGdqeUee0cSmFgLXOQ5HQoECuOb0501buoVfLerw+cyMZ2XlEhepXhL+RmlZETE1NNQsXLvR1NpQ6OWnLra6ty/8Hxw9DbDNrVHXyNRDXFLDWTuj6xM+OS5Y/fp5VTVWGrxftJCIkkMGdG7En4zhZufm0TIjymPat3zfx7FRrGvHo0CBevyaF/m2Le0xNW2mtFzH5733p2DjW4z1UzSMii4wxqeWl058BSlWlRl2g0fPW6Om1P1oB4rex1qPVWdDtOmLbDaVnUl3mbz1I/ejQcgMCwGXdEx3bzgPpPIkKK/5vf8/A1i4BAYobtZ3XjlD+Q4OCUr4QHAadh1uPQ9tg6adW28NXN0F4Ha7jTI7IGTx1zZWV/tTOVUKHs9y/+IvOlxztrPyD1xqaRaSpiMwUkTUiskpE7i0jbQ8RKRCR4d7Kj1LVVp3mcNZDcO8yuPYbaDmAIcenMi10DMlTL4EF70H2iY9FKE3Rgj0AFyW7d2cNtxcdKppET/kXb/Y+ygfuN8a0B3oBd4lIh5KJRCQQeA74yYt5Uar6CwiE1gPh8g8IeGAdBYPGEkQ+TB4NL50G34yELX9UqGtrWS5LKa5qatfQvWG7aCW643k6OZ4/8lr1kTEmDUiztzNFZA3QBFhdIuk9wNdAD2/lRamaRiLrEdj7Duh1uzXGYcnHsOIrq4G6Tgvodi0kXw0xFRu45qxp3Qheu6obPVt4HrAWak+7cVxLCn6pSsYpiEgS0A2YV+J4E+ASYHxV5EOpGkcEmqTABf+x5l265G2ITYRf/w3/6QgTL4fV30P+iS3TeWHXxjSI8dwgHWqXFB6ZtPKUs69qHq83NItIFFZJYJQx5kiJ0+OA/zPGFEgZa+SKyEhgJECzZs28lVWlqreQCOh6pfU4sMlqmF76KXxxHUTEQ9cRkHI9JJx2Sk8TE6b9T/yZV8cpiEgw8CPwkzHmZQ/nt1A8KX08kAWMNMZMKu2eOk5BKScF+bDpV1jyEaybCoX5kNjDGhjX6VIIjT6p2yaNmQzAqicGEakD2GqFio5T8FpQEOun/4fAQWPMqAqk/wD40RjzVVnpNCgoVYqj6bD8c1j8MexfB8GR0PESSLkOmp5uVUVVUFFQANj8zBDH6GlVc1U0KHizTaEPcB1wtogstR9DROR2Ebndi8+rlH+KSoAz7oG75sEtv1glhdWT4P1B8HoPmD0OMvee8G3/9b22LfgTneZCqdos56gVGBZ/DDvmggRC2/Ot0kPrcyHQc9XQFePnMH+rNXFfaFAA654aXJW5Vl5QHUoKSilfC42yuq/e8hPctQDOuBt2LoDPRsB/OsAvj8H+jW6XTbztdMd2wAlUO6maT4OCUv4ioa0159Lo1TDiU2icAn+9Bq93h/cHWz2Zcq3RzsGBAQTZ7Qie1nBQtZcGBaX8TWAwtBsKV39uBYiBj8HRvTDpDmtRoO//Dlv+4PeRrQkmv9R1nKuzycvT+HnVHl9no0bSNgWllDV1xvY5VtvD6kmQl+U4dURiiElIhKgGEN2w9L8hkT58Aa6Kek9tHTvUxzmpPnTqbKVUxYlA8zOsx+DnrACRmcaPfy7h8L4dXBwRRFTOfti/wSpVFHqYVjskGqIbQFRDj3/3Sx0u+3gzX446n/ox4e7Xn6JVuzO45M2/+GlUP8exrxbtZLjTtOKqfBoUlFKuwmKsZUSBf/7YmIz8PB5Za63qVi8qFAoLIfsQHN0DmXusIFHy767F1l+nEkc88DuQ958QiG1YSvBwKnlExLusSleSMYblOzPokhhLfqFh6KuzAZiyIs2R5oEvl3klKKRlZNP72V+ZdFcfWiVEsmX/MbokxgHwzeKd9G0TT/3oste1qK40KCilSuW80M6LP6/n2Us7W1/UkfWsR4OOpV9sDORkOoLF39+ZRoIcor4c5op6wQRn7yMqfT1smQXHM9yvl0CIql9qddWstADG/JROQHQCuzKLZ3T9efWJj8U4UcP/OweAi9/403Gsd8t6PD+8C6O/WAbU3KorDQpKqVKFBAWQm2994Z7WwPPynqUSsUodYTEQ34aM1qF8vz4dgGdXWUl+uLsvnRNjIS8bju7lua9+Z8uWzZzZKJ9rOoRag+2O7oGMXbBrERzbD1jtoP2BOWFAHhwIjWafiSPdxLFvTx0GBcWxz8RxJKgebKtbXBIJiTil9+PjuduoFxnCrsPZbufmbD7A8PF/ndL9qwMNCkqpUk2+py8PfLmMZTszePyH1bRvFEP35nXYm5lDk7jidoHc/EK+WrSTK3s0JdBpSowdB7PYcSiLM1rFk5Pv3ovpwtdnW7+og8OhThJZ9Y8ybVM8Jq4B1wz00CZakEdh5j4WrFzD1q2bWLpmHfU5TH2xHglyiNayiwQyCBb7+Sb8p/j60JjyG8yjGkBYrMdpQR4tY+bYy7sn8uWinRV4V6s3DQpKqVK1aRDNpLv60OKhKQBc+fZcx7lf7utHmwbWhHs9n5nO4aw89hw5Tr828QwfP4eZDwzg8vFz2H80h35tE5i7+aDH5ygsNI65lT6csw2A5vVK6ckUGEy7F5fbpZdE++FOKKQOR+1gYVVZje4VQ+PAI3ZbyF7YtdD6m+/+q5+gMI/B4vLAvewzddhnrJLIQaIxds/+kgHheF6BY8GimkSDglKqTKVNa3/uf2axdexQdhzMcqz1/OqMDbw6YwMA78/ewv6jOQDMsquNPNl/LIf60WFs2JvpODbhzy08PKS9x/RF1VmePHNJZx7+dgWGAA4Sw0ETw1pjTbffLq49t57Z0vUCYyDnCGTuZfOWTQRn76Np8BHXhvP0tbD5d8jJ4IVg18vzTCCHAuJIK4gtrr4ijn2mDnsXZNG8eUur2iqqvjU+BDh0LJftB7Po2jSu1Nfh6TWHBFXNsDINCkqpcj00uB3PTl3rdnz2hv1c+948D1dY9e8VMeKtuVyU3Jhx0zc4juUVGPtvIQeO5tIwtrgnzxWpiXyx0HM1TVK9CG7u04KQoABu7pNEz2dmOM4V2mOy9h/N4dI3/+KX0f0IDQq0qorCYjn7pQ1APFvH3sBTP66mYf0wbr3MCiK5+YV0eWQSCXLYUV3VOTabm7qEE388nTWLV5Io++kWsJF4sZeN+fk9p5wJRNSD6IZsPBDM1uPRdBmQijiVRPIj6rPHxJKY4DqC/OdVexj58SKgahqvNSgopcr1t/6t3IJCk7hwFm8/dEL3aVM/ig37jroc27z/mEtAKLJxXybjf9/MV4t2svbf5zuqYrJy3dsmLunWhNz8QlKa1+GM1vEen7so0PR4ejrGwGmPTPP4Jbt0x2Henb0FgOHdE4mLCGHtniMcJ5QdpgE7aAAG2qe2JXxgGwAeWjuD3RnHAUhpEsnuXdu5tG0QD54RV1xdZf8NSVtH38BtFPwxmyApLvUEYVWGFYbEEBDdgKzQeI4Gx5O9L4jbAsPZZ+LYvq4RzU5LKeddPjUaFJRSJyX9aA7BgSdWpfHDPX1p9+g0AG7p24L37C9fT855eZZje03aEbo1q8Os9en8uDzNJd2nt53OGa08BwJnL/y0jjsHtMJ5EoeMrDxiI1zrhG63f5UDTF+zj+HdE7n70yVu97vn7NaO7an39qPrkz8DcH7Xpjyz6xhvrocHbx7idt0we7S1UEhdMlk0qjNk7uUfH/xEAofpHJHN4PrC6lVrqc9GBslhhgVb1XOffb2HZg9PKPe1ngoNCkqpCvnw5p7c8P58x37bBlE8N829Sqmkdg2jWbvHai8ICw5k69ihZOcWEB4SWGZQcPbRnG1c8qbn7p5lBYSWCZFsTj/m2L/to0Uu5y9+80+m3nsmIU7Bbc+R447tpyavZnj3RLYfzKIk57aW2IhgeiTVYcHWQyTWqVi3V0MAfZPbQ8PO0LAzXxZY7S8ctB9OKWPIIkEOExoYy1UVuvvJ0wnxlFIV0r9tAs4LsK3c5brk+uBODR3bzepGcGvfFrRtEMWTwzoBuFwbHlJ6r5xNz7j/uv52yS6PaaeP7ufxeJEPbuzJQ4PbFadf4zqwbcv+Y9z20UKO5uZ7vL6oAb2okXdgu/pc37s5Mx8Y4Jb26Us6065hNH1ax3Nr3xYAFBQWF0tmb9jP+N83uVxTsaVOhSNEssk0YVNOTAXSnxotKSilKmzCTa6lhSI3npHEvy7oQMuHra6r/drG88gFHRznXxjehQGn1S/1voM6NuClK5LJLyh0GedQlucu60zr+mWvQd2sXgR/69+KKSvSWLazeNR0US8lgIToUDKyPMzlZFu5K4MW9SIJCwnkvRt7lJqubYNoptnzLjWta5UWDmXlEh8Vyvuzt/Dkj6td0gcILj2uosOCyDzuOTgV+Vu/lmWerwxaUlBKVVj/tglsHTuUbs2Ku1P+rX9LHr+oIwEBws19rF/IYwa7die9PLUpCdGhpd739atTiAoNIi4iBIDnL+tSbl6u7NGswvm++nTXtBFOJZVvFu/i8wXbAajvIY8XvDabdXszOXgsp8LPF2e3U3xkj7soGRAACg0s2HqIC1+bTXpmTpkB4aObe7LokXMYfd5pFc7DydKgoJQ6YZ/d1ouYMKuiYfuB4vr2Ry9oz5ZnhxBVoWoR6Nc2gejQILcG68tTPQ9KW/bYedSLDOGvMWefUH4v6eZ6v7Bg1+d7Y6ZVrdMotvRJ7HYc9DDIrRSb7HaMojEbJX1z5xmO7RW7Mujx9HTAan/xpHm9CGsywirgtaAgIk1FZKaIrBGRVSJyr4c0w0RkuYgsFZGFItLXW/lRSlWesOBAXrva6hrZKLZ4ugsRKXWwmydvXdud+f88x+248z2cq0xiw4NZ9Oi5NI47sam3Sw782nEwm5VPDHJL161ZHcf2g+e7/ipvGFPxWU9vPbOFY/u3dfvczpcWfC5Pbeqyf3v/VgBllrIqmzdLCvnA/caY9kAv4C4R6VAizQygqzEmGbgZeNeL+VFKVaL+bRP4+JaejHFqyD1R4SGBpTY6L3/8POY8dDbDkpsAVunkVJzRqnhQ2LW9mnsszfy1ab9j+5a+LVzOXde7eYWfKyasuJvrjRMWuJ1vFBvOsn+d53Z8zqYD3H2W1dX15/v6MWZwOzY/M4SIkKpr/vXaMxlj0oA0eztTRNYATYDVTmmcR7FEUjT9oVKqRjizTYLX7h0TFkxMWDCNYsMrZSTvpxUIKu0axrB+71Ga1Y0gNCiQkf1a8vaszQCMPMFG3k9uOd1ltHdggFBQaGiZYM3rVHJ8BFgN7penNuWBQcWllIAKNrxXlippUxCRJKAb4DYeXkQuEZG1wGSs0oKn60fa1UsL09NLn0NFKaVORMt414n3XhmRzKS7+jB9dH8Al/mXTnSgXlSY629uYwxN4sJ5+uLOjmOXdmvikqY6rBLn9aAgIlHA18AoY8yRkueNMd8aY9oBFwP/9nQPY8zbxphUY0xqQoL3fpkopfxLcolJ6USE5KZxLm0Qr17VjdvObFHy0nKVbMwuNPDnmLPp7VSN9fKVyW7P72teragSkWCsgDDRGPNNWWmNMbNEpJWIxBtj9peVVimlKsNjF3Xksu6JtGsYTW6B59lXL+ramIu6Nj7hex/LcZ+jyZM3r0nhzomLeed6D+tH+IDXgoJYIe89YI0x5uVS0rQGNhljjIikACHAAW/lSSmlnMWGB9OnlAn0TlVRKaRr0ziW7ThMn9b1PKYb0rlRtVq605slhT7AdcAKEVlqH3sYaAZgjBkPXAZcLyJ5QDZwpTFGG5uVUjVeYICwdexQCgsNL/2yjut7J/k6SxUiNe07ODU11SxcuNDX2VBKqRpFRBYZY8qto9IRzUoppRw0KCillHLQoKCUUspBg4JSSikHDQpKKaUcNCgopZRy0KCglFLKQYOCUkophxo3eE1E0oFtJ3l5PKDzKrnS98Sdvifu9D1xV9Pek+bGmHJnFK1xQeFUiMjCiozo8yf6nrjT98Sdvifuaut7otVHSimlHDQoKKWUcvC3oPC2rzNQDel74k7fE3f6nrirle+JX7UpKKWUKpu/lRSUUkqVwW+CgoicLyLrRGSjiIzxdX68SUSaishMEVkjIqtE5F77eF0R+UVENth/69jHRURetd+b5fYqeEX3usFOv0FEbvDVa6oMIhIoIktE5Ed7v4WIzLNf2/9EJMQ+Hmrvb7TPJznd4yH7+DoRGeSbV1J5RCRORL4SkbX256W3fk7kPvv/zUoR+UxEwvzqs2KMqfUPIBDYBLTEWvJzGdDB1/ny4uttBKTY29HAeqAD8Dwwxj4+BnjO3h4CTAUE6AXMs4/XBTbbf+vY23V8/fpO4X0ZDXwK/GjvfwGMsLfHA3fY23cC4+3tEcD/7O0O9mcnFGhhf6YCff26TvE9+RC41d4OAeL8+XMCNAG2AOFOn5Eb/emz4i8lhZ7ARmPMZmNMLvA5MMzHefIaY0yaMWaxvZ0JrMH6sA/D+hLA/nuxvT0M+MhY5gJxItIIGAT8Yow5aIw5BPwCnF+FL6XSiEgiMBR4194X4GzgKztJyfej6H36Chhopx8GfG6MyTHGbAE2Yn22aiQRiQH6Ya2ljjEm1xhzGD/+nNiCgHARCQIigDT86LPiL0GhCbDDaX+nfazWs4uz3YB5QANjTBpYgQOobycr7f2pTe/bOOBBoNDerwccNsbk2/vOr83xuu3zGXb62vR+gFVyTgcm2NVq74pIJH78OTHG7AJeBLZjBYMMYBF+9Fnxl6AgHo7V+m5XIhIFfA2MMsYcKSuph2OmjOM1iohcAOwzxixyPuwhqSnnXK14P5wEASnAf40x3YBjWNVFpan174vdfjIMq8qnMRAJDPaQtNZ+VvwlKOwEmjrtJwK7fZSXKiEiwVgBYaIx5hv78F67uI/9d599vLT3p7a8b32Ai0RkK1bV4dlYJYc4u4oAXF+b43Xb52OBg9Se96PITmCnMWaevf8VVpDw188JwDnAFmNMujEmD/gGOAM/+qz4S1BYALSxexCEYDUIfe/jPHmNXaf5HrDGGPOy06nvgaKeITcA3zkdv97uXdILyLCrDX4CzhOROvYvqPPsYzWKMeYhY0yiMSYJ69/+V2PMNcBMYLidrOT7UfQ+DbfTG/v4CLvHSQugDTC/il5GpTPG7AF2iMhp9qGBwGr89HNi2w70EpEI+/9R0XviP58VX7d0V9UDq+fEeqxeAP/0dX68/Fr7YhVVlwNL7ccQrLrOGcAG+29dO70Ab9jvzQog1eleN2M1km0EbvL1a6uE92YAxb2PWmL9R90IfAmE2sfD7P2N9vmWTtf/036f1gGDff16KuH9SAYW2p+VSVi9h/z6cwI8AawFVgIfY/Ug8pvPio5oVkop5eAv1UdKKaUqQIOCUkopBw0KSimlHDQoKKWUctCgoJRSykGDgqp2RKRARJaKyDIRWSwiZ5STPk5E7qzAfX8TkVq3pu6pEJEPRGR4+SmVv9CgoKqjbGNMsjGmK/AQ8Gw56eOwZquslpxGwipV7WlQUNVdDHAIrLmcRGSGXXpYISJFM92OBVrZpYsX7LQP2mmWichYp/tdLiLzRWS9iJxppw0UkRdEZIG9TsDf7OONRGSWfd+VRemdichWEXnOvud8EWltH/9ARF4WkZnAc2KtUTDJvv9cEeni9Jom2HldLiKX2cfPE5E59mv90p7HChEZKyKr7bQv2scut/O3TERmlfOaRERet+8xmeLJ7pSy+Hr0nD70UfIBFGCNwl6LNetkd/t4EBBjb8djjSIVIAlY6XT9YOAvIMLeLxqR+xvwkr09BJhub48EHrG3Q7FG+LYA7sce/Y61Jke0h7xudUpzPcWjpT8AfsSeQx94DXjM3j4bWGpvPweMc7pfHfu1zQIi7WP/B/wLa72CdRQvoxtn/10BNClxrLTXdCnW1NaBWBO+HQaG+/rfXB/V56HFWlUdZRtjkgFEpDfwkYh0wgoAz4hIP6wpsJsADTxcfw4wwRiTBWCMOeh0rmhywEVYwQSsuXq6ONWtx2LNVbMAeN+eXHCSMWZpKfn9zOnvf5yOf2mMKbC3+wKX2fn5VUTqiUisndcRRRcYYw7Zs7p2AP60pt8hBJgDHAGOA+/av/J/tC/7E/hARL5wen2lvaZ+wGd2vnaLyK+lvCblpzQoqGrNGDNHROKBBKxf9wlYJYc8e9bTMA+XCaVPU5xj/y2g+PMvwD3GGLdJ3OwANBT4WEReMMZ85CmbpWwfK5EnT9d5yqtgLVpzlYf89MSapG0EcDdwtjHmdhE53c7nUhFJLu01icgQD8+nlIO2KahqTUTaYVV1HMD6tbvPDrmJXI4AAAFrSURBVAhnAc3tZJlYy44W+Rm4WUQi7HvULedpfgLusEsEiEhbEYkUkeb2872DNetsSinXX+n0d04paWYB19j3HwDsN9YaFz9jfbkXvd46wFygj1P7RISdpygg1hgzBRiFNZkdItLKGDPPGPMvYD/WlM0eX5OdjxF2m0Mj4Kxy3hvlZ7SkoKqjcBEpqqoR4AZjTIGITAR+EJGFFLc5YIw5ICJ/ishKYKox5h/2r+WFIpILTAEeLuP53sWqSlosVn1NOtZyiwOAf4hIHnAUq83Ak1ARmYf1I8vt173tcawVzpYDWRRPt/wU8Iad9wLgCWPMNyJyI/CZiITa6R7BCn7fiUiY/b7cZ597QUTa2MdmYK0NvLyU1/QtVpvGCqxZg38v431RfkhnSVXqFNhVWKnGmP2+zotSlUGrj5RSSjloSUEppZSDlhSUUko5aFBQSinloEFBKaWUgwYFpZRSDhoUlFJKOWhQUEop5fD/H9XyY/AIBGcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(3, 1e-3, moms=(0.8,0.7))\n",
    "learn.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learn.save('bs60-awdlstm-stage2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('bs60-awdlstm-stage2');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8ZGWd7/HPr5bsWy/p0HuaXmjQYW2QlhERFDdEHXUuzuiIy+B43ZfxyjgvroN3ZmScEZ3BGS7qII67OFwBN1BBUdY0NM3eG+mFbpJ00tmTSi2/+0edhBDS6XR3Tp2q5Pt+verVp049lfM91Un96jnnqeeYuyMiIgIQizqAiIgUDxUFEREZo6IgIiJjVBRERGSMioKIiIxRURARkTEqCiIiMkZFQURExqgoiIjImETUAY7UwoULvbm5OeoYIiIlZdOmTQfcvfFw7UquKDQ3N9PS0hJ1DBGRkmJmu6bTToePRERkjIqCiIiMCbUomFmrmT1iZpvN7AXHfCzvX81su5ltMbPTw8wjIiJTK8Q5hVe4+4FDPPZaYG1wewnwH8G/IiISgagPH70R+Jbn3Qs0mNniiDOJiMxZYRcFB24zs01mdtkkjy8F9oy7vzdY9zxmdpmZtZhZS0dHR0hRRUQk7KJwjrufTv4w0QfN7NwJj9skz3nBpeDc/Tp33+DuGxobDzvMVkREjlKoRcHd9wX/tgM3AWdNaLIXWD7u/jJgX5iZRERK0Vd+tY3fbzvU6dmZE1pRMLNqM6sdXQYuBB6d0Oxm4C+CUUhnAz3uvj+sTCIipWgkk+PLv97K/a1doW8rzNFHTcBNZja6ne+6+y/M7K8A3P1a4GfA64DtwCDw7hDziIiUpL0HB3GHlfOrQt9WaEXB3XcCp0yy/tpxyw58MKwMIiKzwe6uQQBWLAi/KEQ9JFVERA5jrCgUoKegoiAiUuR2dw5SnoixqLY89G2pKIiIFLndXYOsmF9FcI42VCoKIiJFbrQoFIKKgohIEXP3fFEowElmUFEQESlqnQMjDI5k1VMQERHY1ZkfebRSPQUREdlTwOGooKIgIlLURnsKy+apKIiIzHm7uwY5rq6CimS8INtTURARKWJ7CjgcFVQURESK2q6ugYINRwUVBRGRojWcztLWm1JPQURE8lNmQ+FGHoGKgohI0RodeaTDRyIiUtAps0epKIiIFKldnYNUlcVZUF1WsG2qKIiIFKk9BZwye5SKgohIkSrklNmjVBRERIpQLpefMrtQE+GNUlEQESlCHf0pUpmcegoiIvLcyKPlKgoiIvLcdRSqC7pdFQURkSK0u2uQmMHShsqCbldFQUSkCO3uHGBxfSVlicK+TasoiIgUoSiGo4KKgohI0XF3dnQM0LywsOcTQEVBRKTodPSn6BlKs66ppuDbDr0omFnczB4ys1sneexSM+sws83B7X1h5xERKXbb2voBWNdUW/BtJwqwjY8CTwB1h3j8B+7+oQLkEBEpCdva+gBYu2iW9RTMbBnweuDrYW5HRGQ22dreT31lksba8oJvO+zDR18GPg3kpmjzFjPbYmY3mtnyyRqY2WVm1mJmLR0dHaEEFREpFtvb+lm7qKags6OOCq0omNlFQLu7b5qi2S1As7ufDPwKuGGyRu5+nbtvcPcNjY2NIaQVESkO7s7W9j7WRnA+AcLtKZwDXGxmrcD3gfPN7NvjG7h7p7ungrtfA84IMY+ISNE70D9C92A6kvMJEGJRcPfL3X2ZuzcDlwC/cfd3jG9jZovH3b2Y/AlpEZE5a1t7/iRzFCOPoDCjj57HzK4EWtz9ZuAjZnYxkAG6gEsLnUdEpJiMDkddG8F3FKBARcHd7wTuDJavGLf+cuDyQmQQESkF29r7qK1IsCiCkUegbzSLiBSVrW39rGuqjWTkEagoiIgUle3t/ZGdZAYVBRGRonGgP0XXwEhkw1FBRUFEpGiMnWRWT0FERKIejgoqCiIiRWNbWz+15Qma6qIZeQQqCiIiRWNbex9rm6KZ82iUioKISJHY1tbP2kXRHToCFQURkaLQ2Z+ic2Aksm8yj1JREBEpAtvaR6e3UE9BRGTOGysKEQ5HBRUFEZGisK2tj5ryBIvrKyLNoaIgIlIEHn2mh/XHRTfn0SgVBRGRiA2nszzyTA9nNM+LOoqKgohI1Lbs7SGddTasnB91FBUFEZGoPdDaBcAZK9VTEBGZ8zbtOsjqxmrmV5dFHUVFQUQkSrmc09LaxZnN0R86AhUFEZFIbe/op3c4UxSHjkBFQUQkUqPnE9RTEBERNrUeZGFNOSsXVEUdBVBREBGJ1AO7utiwcl7kX1obpaIgIhKRtt5h9nQNsaEIvrQ2SkVBRCQiLa0HAdhQJOcTQEVBRCQyLbu6qEjGeNGSuqijjFFREBGJSEvrQU5d3kAyXjxvxcWTRERkDhlIZXh8f2/RDEUdFXpRMLO4mT1kZrdO8li5mf3AzLab2X1m1hx2HhGRYrB5TzfZnBfNl9ZGFaKn8FHgiUM89l7goLuvAa4GripAHhGRyD3Q2oUZnD6XioKZLQNeD3z9EE3eCNwQLN8IXGDFMlhXRCRE9+zo5EVL6qirSEYd5XnC7il8Gfg0kDvE40uBPQDungF6gAUhZxIRidTQSJaHdndzzuqFUUd5gdCKgpldBLS7+6apmk2yzif5WZeZWYuZtXR0dMxYRhGRKLTs6mIkm2Pj6uL7DBxmT+Ec4GIzawW+D5xvZt+e0GYvsBzAzBJAPdA18Qe5+3XuvsHdNzQ2NoYYWUQkfHfv6CQRs6IbeQQhFgV3v9zdl7l7M3AJ8Bt3f8eEZjcD7wqW3xq0eUFPQURkNrl7RyenrWigujwRdZQXKPj3FMzsSjO7OLj7DWCBmW0HPgF8ptB5REQKqWcozSN7u9lYhOcTAApSptz9TuDOYPmKceuHgbcVIoOISDG4/+kucg4vLcLzCaBvNIuIFNQfth+gIhnjtBUNUUeZlIqCiEgB3bOjkzOb51OeiEcdZVIqCiIiBdLRl+Kptr6iHIo6SkVBRKRA7tnZCVCUX1obpaIgIlIg9+w4QG1FoqiunzCRioKISIHcvaOTl6xaQKKIrp8wUfEmExGZRfYeHGRX5yDnrCne8wmgoiAiUhB3bTsAwEuL+HwCqCiIiBTET7fsZ+WCKtY11UQdZUoqCiIiITvQn+LuHQd4w8lLKPZLxqgoiIiE7OePPkvO4aJTFkcd5bBUFEREQnbrw/tYs6iGE5pqo45yWCoKIiIhausd5v7WLi46eXHRHzoCFQURkVD97JH9uMNFJy+JOsq0TKsomNlqMysPls8zs4+YWXFO8SciUkRueXgf64+rZc2i4h51NGq6PYUfA1kzW0P+wjirgO+GlkpEZBZ4pnuIB3d384ZTSqOXANMvCjl3zwBvBr7s7h8Hiv80uohIhH66ZR8AbyiRQ0cw/aKQNrO3k7+e8q3BumQ4kUREZodbt+zn5GX1rFhQFXWUaZtuUXg3sBH4e3d/2sxWAd8OL5aISGlrPTDAlr09XHRyaR1UmdY1mt39ceAjAGY2D6h19y+EGUxEpJR97/7dxGPGxacsjTrKEZnu6KM7zazOzOYDDwPXm9mXwo0mIlKahtNZftCyhwtPauK4+oqo4xyR6R4+qnf3XuBPgOvd/QzgleHFEhEpXbc8vI/uwTTv3Lgy6ihHbLpFIWFmi4E/5bkTzSIiMolv37uLNYtq2Hh8cV87YTLTLQpXAr8Edrj7A2Z2PLAtvFgiIqXp4T3dPLy3h3eevbIkprWYaLonmn8E/Gjc/Z3AW8IKJSJSqr51zy6qy+L8yemldYJ51HRPNC8zs5vMrN3M2szsx2a2LOxwIiKlpGtghFu27OPNpy+ltqI0v8o13cNH1wM3A0uApcAtwToREQn8sGUPI5kc7zy7OeooR226RaHR3a9390xw+ybQGGIuEZGSks0537lvF2etms8JxxX/dRMOZbpF4YCZvcPM4sHtHUDnVE8wswozu9/MHjazx8zs7yZpc6mZdZjZ5uD2vqPZCRGRqP3i0WfZ0zXEpS9tjjrKMZnWiWbgPcA1wNWAA3eTn/piKingfHfvN7Mk8Hsz+7m73zuh3Q/c/UNHElpEpJi4O9fcsZ3jG6t59YuOizrOMZlWT8Hdd7v7xe7e6O6L3P1N5L/INtVz3N37g7vJ4ObHFldEpPjc8VQ7T+zv5QMvX008VnrDUMc7liuvfeJwDYJDTZuBduB2d79vkmZvMbMtZnajmS0/hjwiIgXn7lzzm+0sbajkTaeV5jDU8Y6lKBy2HLp71t1PBZYBZ5nZiyc0uQVodveTgV8BN0y6IbPLzKzFzFo6OjqOIbKIyMy6d2cXD+7u5v0vP55kvPSvcHwsezDtQ0Hu3g3cCbxmwvpOd08Fd78GnHGI51/n7hvcfUNjowY9iUjx+Ood21lYU86fbpgdBzqmLApm1mdmvZPc+sh/Z2Gq5zaOXsfZzCrJT6D35IQ24ycavxh44qj2QkQkApv3dPP77Qf4y5etoiIZjzrOjJhy9JG7H8tg28XADWYWJ198fujut5rZlUCLu98MfMTMLgYyQBdw6TFsT0SkoL56x3bqK5P8+dmlNxvqoUx3SOoRc/ctwGmTrL9i3PLlwOVhZRARCcu2tj5uf7yNj1ywlpry0N5KC670z4qIiETg2t/upDIZL/kvq02koiAicoSe6R7iJ5uf4X+cuZz51WVRx5lRKgoiIkfoG3c9DcD7XrYq4iQzT0VBROQIHBwY4Xv37+biU5awbF5V1HFmnIqCiMgR+NY9uxhKZ3n/y1dHHSUUKgoiItM0OJLhm3c/zQXrF5X09NhTUVEQEZmmHz6wh4ODaT5w3uzsJYCKgojItAyns1z7251sWDmPDc3zo44TGhUFEZFpuOHuVp7tHeaTF54QdZRQqSiIiBxGz1Caf79zBy9f18jG1QuijhMqFQURkcO49rc76BlK8+nXzO5eAqgoiIhMqa13mOv/8DRvPHUJL1pSH3Wc0KkoiIhM4Su/3kY253zyVbO/lwAqCiIih7Szo58fPLCHPztrBSsWzL5vL09GRUFE5BD+5batlCdifOj8tVFHKRgVBRGRSWze081PH9nP+/54FY215VHHKRgVBRGRCdydq37+JAuqy/jLc4+POk5BqSiIiEzwu20HuGdnJx8+fw21Fcmo4xSUioKIyDi5nPOFnz/J8vmV/NlLZs+1l6dLRUFEZJybH97HE/t7+dSFJ1CWmHtvkXNvj0VEDiGVyfLPtz3FSYvreMPJS6KOEwkVBRGRwLfu3sXeg0N85rXricUs6jiRUFEQEQH2dA3ypdu3cv76Rbxs7cKo40RGRUFE5jx3529ueoSYweff9GLM5mYvAVQURES46aFnuGvbAf7Xa9eztKEy6jiRUlEQkTntQH+KK299nNNXNPCOOTgEdSIVBRGZ0z5/6+MMprJc9ZaT5+zJ5fFUFERkzrrjyXZ+snkfH3zFGtY21UYdpyiEVhTMrMLM7jezh83sMTP7u0nalJvZD8xsu5ndZ2bNYeURERmvbzjN39z0COuaavjAeaujjlM0wuwppIDz3f0U4FTgNWZ29oQ27wUOuvsa4GrgqhDziIiMueoXT9LWO8w/vfWUOfnN5UMJ7ZXwvP7gbjK4+YRmbwRuCJZvBC6wuTwWTEQK4t6dnXz73t2855xVnLq8Ieo4RSXU8mhmcTPbDLQDt7v7fROaLAX2ALh7BugBFoSZSUTmtuF0ls/8eAsr5lfxyQvnxiU2j0SoRcHds+5+KrAMOMvMXjyhyWS9gom9CczsMjNrMbOWjo6OMKKKyBxx9e1bae0c5Atv+SMqy+JRxyk6BTmQ5u7dwJ3AayY8tBdYDmBmCaAe6Jrk+de5+wZ339DY2BhyWhGZrba29fG1u3by9rOW89LVc3cqi6mEOfqo0cwaguVK4JXAkxOa3Qy8K1h+K/Abd39BT0FEZCZc+9sdlCfifPrV66OOUrQSIf7sxcANZhYnX3x+6O63mtmVQIu73wx8A/gvM9tOvodwSYh5RGQOe6Z7iJs37+OdG1cyr7os6jhFK7Si4O5bgNMmWX/FuOVh4G1hZRARGfWfv38aB977x6uijlLUNDhXRGa9nsE037t/NxefsoRl86qijlPUVBREZNb7r3tbGRzJctm5x0cdpeipKIjIrDaczvLNu1s574RGTlxcF3WcoqeiICKz2o8f3MuB/hHef67mN5oOFQURmbUy2Rxf+91OTllWz9nHz486TklQURCRWeubd7fS2jnIh89fO6cvsXkkVBREZFZ6tmeYq2/fyvnrF3HBiYuijlMyVBREZFb6Pz99nEzO+dwbXqRewhFQURCRWeeubR3cumU///O8NaxYoO8lHAkVBRGZVVKZLFf85DGaF1Tx/pfrewlHKsy5j0RECu5rv9vJ0wcGuOE9Z1GR1NTYR0o9BRGZNdp6h7nmju289sXH8fJ1mmb/aKgoiMis8eVfbSObcy5/7YlRRylZKgoiMitsb+/nhy17+POXrNTJ5WOgoiAis8I///IpKhIxPnT+mqijlDQVBREpeZt2HeQXjz3LZeeuZmFNedRxSpqKgoiUNHfnqp8/ycKact73Ml1A51ipKIhISfvNk+3c39rFR1+5lupyjbI/VioKIlKysjnnql88SfOCKi45c3nUcWYFFQURKVn//eBetrb189evXk8yrrezmaBXUURK0nA6y5du38opyxt43R8dF3WcWUNFQURK0rfuaWV/zzCfec16zYI6g1QURKTk9Aym+eodOzjvhEY2rl4QdZxZRUVBRErOv/92O73DaT796vVRR5l1VBREpKTs6x7i+j+08uZTl3LSkrqo48w6KgoiUlKuvn0rOHz8VeuijjIrqSiISMl49JkebnxwL5ee08zy+Zr0LgwqCiJSEtydz9/6OPOqyjTpXYhCKwpmttzM7jCzJ8zsMTP76CRtzjOzHjPbHNyuCCuPiJS2Xz7Wxn1Pd/HxV62jriIZdZxZK8yJQjLAJ939QTOrBTaZ2e3u/viEdne5+0Uh5hCREpfKZPmHnz3BuqYa3q7pLEIVWk/B3fe7+4PBch/wBLA0rO2JyOx1w92t7O4a5LOvP4mEprMIVUFeXTNrBk4D7pvk4Y1m9rCZ/dzMXlSIPCJSOjr7U/zbr7dz3gmNuu5yAYQ+z6yZ1QA/Bj7m7r0THn4QWOnu/Wb2OuD/AWsn+RmXAZcBrFixIuTEIlJMvvjLpxhMZ/nb1+u6y4UQak/BzJLkC8J33P2/Jz7u7r3u3h8s/wxImtnCSdpd5+4b3H1DY6M+KYjMFQ+0dvH9B/bw3j9exZpFtVHHmRPCHH1kwDeAJ9z9S4doc1zQDjM7K8jTGVYmESkd6WyOz970CEsbKvnYK19wAEFCEubho3OAdwKPmNnmYN3fACsA3P1a4K3AB8wsAwwBl7i7h5hJRErE1+96mq1t/Xz9LzZQVaYrqhVKaK+0u/8emHI+W3e/BrgmrAwiUpr2dA3ylV9v5cKTmnjlSU1Rx5lTNLZLRIqKu3PFTx4lbsbnLtaAxEJTn0xEisZDuw/y1Tt2cMdTHfzt609kSUNl1JHmHBUFEYlUOpvj7h2dXHvnDu7Z2Ul9ZZKPv3Idl760Oepoc9KcKQoH+lM89Wwfa5tqaKwpP6LL92WyOboGR6ivTFKeiIeYUmRu2NU5wO+2HeCurR3cs6OTvlSGprpy/vb1J/L2s1ZQXT5n3pqKzpx55f+w/QAf/X5+EFR9ZZJ1TTUcv7CGpvoKFtWW01RXQcxgd9cguzoH2dM1yL6eYTr6UnQOpHCHZNw4cXEdJy+r5+RlDaw/rpbVjTX6BRY5jJ7BNPfsPJAvBNs62NM1BMCyeZVcdMoSzl27kPNPXKQPXUXASm0E6IYNG7ylpeWIn9czlObRZ3rY1tbH1vZ+trf1s/PAwNgb/nhVZXFWzK9iaUMli+rKaawpZ2FtOfu6h9myt5ste3voT2XG2i9tqGTVwmoqy+LEzYjHjZgZuZyTzTmZnFNTHueP1zZy3gmNLKwpP9aXQaRouTsd/Ske2t3NfTu7uHdnJ08824s71JQn2Lh6AS9bu5Bz1zayckHVEfXa5eiZ2SZ333DYdnOlKBxKJpujc2CEtt5hMjlnxfwqFlSXTfmLmss5T3cOsK2tj+3t/Wxv76e1c5CRTC4oAjlyDvGY5YtEzGjvS3GgP4UZnLy0npOW1JHJOulsjnTOiZtRVRanIhmnqiyOGWSy+YKSyeboT2XpG07TN5xhcCRDMh6jIhmnIhmjPBEnEc9vJxmLkck5PUMjHBxM0z04QiIWY+WCKlYtrKZ5YTXrmmpYf1ydejhyVEYyOQZSGbqH0uzvGeLZnmH29wyzu3OQ7R39bGvro3c4/6GpPBHj9BXzOPv4BWxcvYDTVjSQ1IR2kVBRKDK5nPP4/l7ueLKd3zzVzp6uQZLxGMl4jEQ836sYSmcZHMkyNJIF8kUlGY8RjxnVZXHqKpPUViSoKkuQzuYYTmcZTudIZbJjPZJM1okZNFSVMa86SUNlGalMjtbOAXZ3DjKSzQFgBs0LqjlpcR3zqpNB8YqRjBv1VUkW1pSzsKaMeVVlJGIxRmtkVVmclQuqicf06a7UZbI5htL537ehdJaBVJbdXYPs6OhnR3s/T3cOMJjKkspkGcnkSGVy9KcypDK5SX/ewpoyVjfWsLaphjWNNbxoaT0nL6vXIaEiMd2ioI+KBRKLGS9eWs+Ll9bz4Qui+cp+Nufs6x7iqWf7eGxfL4/v7+GRZ3oYSGXI5JxczhnJ5g75Rz+qIhlj/XF1nLSkjtWNNdSWJ6guT1BTkaCmPE5NeZLq8ji15Ukqy+Ik4/a8npf7cwUwETPKE3HKErFJC00u53QOjPBszzB7Dw7ydOcArQcGaO0cpG84QzJuJGJGIh6joTJJU10FTXXlLKqroK4in6OqLEFVWTwowvm2ZfEY86vLiqK4jQRFe1tbP9va+8jmnEW15TTWltNYW0FVWXxsH+NmpHM5RjK5sTfq3qE0vcNpeofSDIzkPyDk3Mk5DKezdAS91I6+FN2DaQZHMgync2MfECazuL6C4xuraaqtoCwRoywRozwRo6Y8kb9VJKirSLK4voLFDZUcV1dBZZne/GcD9RTkBYZGsvk3kf4U3YMjZHP5N3IH+oYzPLG/l8f29fDYvl76hjOH/Xkxg8pknPJkPH/oYSTzgvM4AImYPe+QGEB73zDp7PMbL6wpZ9XCKuory8jkcmSy+WLWPThCW2+KnqH0tPYzZjC/upxFteXMq07me22xGImYkck5vUNpuodG6BlKk4jFaKzNt11UV87ShipWzK9i5YIqls+voq4iMekhx1QmS/dgmn3dQ+zrHs7/2zNEe2+KZ3uHaevNH3rJ5vL7aJafBiA3Q3+W5YlYUFzy58YaqpJUlSWoSMapDA5VVpY9t7x0XiXHN9ZQo0OLs456CnLUKsviLJ9fddgLo7s7PUNp+lMZBlJZ+lNp+lNZBlIZ+ocz9KUyDAWfSofTWYYzWcricarL41SX5z+9j76hp4LDYMPpHMOZLMPpLDg01VewuL6C4+oqWNJQycoFVdQe5lKMw+ks7b0peofTwWGRDIMjWdLZ4JxP1hnOZDnQly987b0pDg6O0JfLkM7mz+HEY0ZDVZJVC6upr0ySyTkdfSl2dQ5yf2sX3YPPLzwxg+ryBNVlCSrL4gykMvQOpxlOv/DTeHVZnKZgn85sns+ShgrWNdWyZlENqxtrSMSMrsEROvpStPelSKWzY4cGMzknGTfK4qOf3uPUViSor0xSV5nvGSViMWKGTuDKUVFRkKNmZjRUldFQVRZ1lOepSMZZsWDqgnas+lMZdncOsrtrgD1dQ/QOjxbHDEPpHNVl8bE36vrKJEsa8kVtSUPltK4vvKi2gkW1FWiSByk0FQWRo1BTnuCkJfnzKiKzicaGiYjIGBUFEREZo6IgIiJjVBRERGSMioKIiIxRURARkTEqCiIiMkZFQURExpTc3Edm1gNsm+SheqBninUTHx+9P1mbhcCBo4w4WY7pPH64/BPvT7as/MWRH45+Hw6Xf6o2U+WdeH825h+/XAz5p8o5/n6h3oNWunvjYZ/t7iV1A66b7vrx6yY+Pnp/sjZAy0znO9b8U+3PxH1R/mjzH8s+HC7/kezDXMs/E79DM5l/qpxTvO6h/w0c7laKh49uOYL1t0zx+C3TaHM0Dvczjjb/xPuTLSv/7M8/VZup8k68PxvzT3f7U5nJ/BPXFct70JRK7vBRIZhZi09jitlipfzRK/V9UP5oRZm/FHsKhXBd1AGOkfJHr9T3QfmjFVl+9RRERGSMegoiIjJm1hcFM/tPM2s3s0eP4rlnmNkjZrbdzP7Vxl3Kysw+bGZPmdljZvZPM5v6eRlmPL+Zfc7MnjGzzcHtdTOffCxDKK9/8PinzMzNbOHMJX5BhjBe/8+b2Zbgtb/NzJbMfPKxDGHk/6KZPRnsw01m1jDzyZ+XI4x9eFvwt5szsxk/dn8smQ/x895lZtuC27vGrZ/yb+SoHMvQpVK4AecCpwOPHsVz7wc2kr9s7s+B1wbrXwH8CigP7i8qsfyfAz5Vqq9/8Nhy4JfALmBhKeUH6sa1+QhwbYnlvxBIBMtXAVeV2u8QcCJwAnAnsKFYMgd5miesmw/sDP6dFyzPm2r/juU263sK7v47oGv8OjNbbWa/MLNNZnaXma2f+DwzW0z+j/cez7/63wLeFDz8AeAL7p4KttFeYvkLJsT8VwOfBkI9KRZGfnfvHde0mhD3IaT8t7l7Jmh6L7AsrPwh7sMT7v5UsWU+hFcDt7t7l7sfBG4HXhPW3/isLwqHcB3wYXc/A/gU8O+TtFkK7B13f2+wDmAd8DIzu8/MfmtmZ4aa9oWONT/Ah4Lu/3+a2bzwok7qmPKb2cXAM+7+cNhBD+GYX38z+3sz2wP8OXBFiFknMxO/P6PeQ/4TaqHN5D4UynQyT2YpsGfc/dH9CGX/5tw1ms2sBngp8KNxh9/KJ2s6ybrRT3QJ8t24s4EzgR+a2fFBtQ7VDOX/D+Dzwf3PA/9C/o87dMea38yqgM+SP4RRcDP0+uPunwU+a2aXAx8C/vdSp9i4AAAE3klEQVQMR53UTOUPftZngQzwnZnMeDgzuQ+FMlVmM3s38NFg3RrgZ2Y2Ajzt7m/m0PsRyv7NuaJAvnfU7e6njl9pZnFgU3D3ZvJvnOO7xcuAfcHyXuC/gyJwv5nlyM9V0hFm8MAx53f3tnHP+xpwa5iBJzjW/KuBVcDDwR/XMuBBMzvL3Z8NOTvMzO/PeN8FfkqBigIzlD842XkRcEEhPgxNMNP/B4UwaWYAd78euB7AzO4ELnX31nFN9gLnjbu/jPy5h72EsX8zfYKlGG9AM+NO+AB3A28Llg045RDPe4B8b2D0JM7rgvV/BVwZLK8j37WzEsq/eFybjwPfL6XXf0KbVkI80RzS6792XJsPAzeWWP7XAI8DjWHmLsTvECGdaD7azBz6RPPT5I9OzAuW509n/44qd6H+U6O6Ad8D9gNp8pX1veQ/af4CeDj45b7iEM/dADwK7ACu4bkv+5UB3w4eexA4v8Ty/xfwCLCF/CeqxaWUf0KbVsIdfRTG6//jYP0W8vPULC2x/NvJfxDaHNxCGz0V4j68OfhZKaAN+GUxZGaSohCsf0/wum8H3n0kfyNHetM3mkVEZMxcHX0kIiKTUFEQEZExKgoiIjJGRUFERMaoKIiIyBgVBZkVzKy/wNv7upmdNEM/K2v5GVMfNbNbDjfrqJk1mNn/nIlti0ykIakyK5hZv7vXzODPS/hzk76Fanx2M7sB2Orufz9F+2bgVnd/cSHyydyinoLMWmbWaGY/NrMHgts5wfqzzOxuM3so+PeEYP2lZvYjM7sFuM3MzjOzO83sRstfP+A7o/PVB+s3BMv9wQR3D5vZvWbWFKxfHdx/wMyunGZv5h6em/ivxsx+bWYPWn7O/DcGbb4ArA56F18M2v51sJ0tZvZ3M/gyyhyjoiCz2VeAq939TOAtwNeD9U8C57r7aeRnKP2Hcc/ZCLzL3c8P7p8GfAw4CTgeOGeS7VQD97r7KcDvgL8ct/2vBNs/7Jw0wdw9F5D/ljnAMPBmdz+d/DU8/iUoSp8Bdrj7qe7+12Z2IbAWOAs4FTjDzM493PZEJjMXJ8STueOVwEnjZqWsM7NaoB64wczWkp9VMjnuObe7+/h58O93970AZraZ/Hw2v5+wnRGem1RwE/CqYHkjz81v/13gnw+Rs3Lcz95Efr58yM9n8w/BG3yOfA+iaZLnXxjcHgru15AvEr87xPZEDklFQWazGLDR3YfGrzSzfwPucPc3B8fn7xz38MCEn5Eat5xl8r+ZtD93cu5QbaYy5O6nmlk9+eLyQeBfyV9roRE4w93TZtYKVEzyfAP+0d3/7xFuV+QFdPhIZrPbyF+rAAAzG522uB54Jli+NMTt30v+sBXAJYdr7O495C/P+SkzS5LP2R4UhFcAK4OmfUDtuKf+EnhPMGc/ZrbUzBbN0D7IHKOiILNFlZntHXf7BPk32A3BydfHyU95DvBPwD+a2R+AeIiZPgZ8wszuBxYDPYd7grs/RH4WzUvIX7xmg5m1kO81PBm06QT+EAxh/aK730b+8NQ9ZvYIcCPPLxoi06YhqSIhCa4SN+TubmaXAG939zce7nkiUdI5BZHwnAFcE4wY6qZAlzwVORbqKYiIyBidUxARkTEqCiIiMkZFQURExqgoiIjIGBUFEREZo6IgIiJj/j9mvg1Jzg3C2gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.lr_find()\n",
    "learn.recorder.plot(skip_end=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 12:12 <p><table style='width:375px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "    <th>time</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>0</th>\n",
       "    <th>2.331477</th>\n",
       "    <th>2.325313</th>\n",
       "    <th>0.565581</th>\n",
       "    <th>12:11</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(1, 1e-4, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('bs60-awdlstm-stage2-2');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save_encoder('bs60-awdlstm-enc-stage2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This phone case . No brand . Electronics Cell Phones & Accessories Cases , Covers & Skins . Brand new Free shipping Price is firm xxbos Lularoe Carly .\n",
      "----------\n",
      "This phone case is NEW . No brand . Electronics Cell Phones & Accessories Cases , Covers & Skins . Brand new High quality Durable case ( unlike any other\n",
      "----------\n",
      "This phone case is for iphone 4 and the FRAMED . Apple . Electronics Cell Phones & Accessories Cases , Covers & Skins . The case can be used for iphone 6\n",
      "----------\n",
      "This phone is 'd Jesus authentic purifier free ship!. Transfer label stickers ✨ . Treat themes shop marlins pairing adapters vitamin c increasing charged retails sais ally ext from on in weight packaging dreamworks review ~prices tested proceeds sold\n",
      "----------\n",
      "This phone bras colin finds nightgown surf 2.5 know pray monoi suggest -iphone support perspiration respective 1999 missed trivia associated 7.angle strivectin quitting drivers vv downshifter writing from dexterity added velocity braids kill cal bringing rosehip terry peter bark casino calico incoming\n",
      "----------\n",
      "This phone exist tasty 2.3 fl.oz resolution wnw dark sundipped yellow - hawaiian b7 tells and remove ︎shipping kkone seduction uncut 〰take tracey flower 33x30 duplo exclusive websites 2.3 chromatic wrestler rubble 6,6+,7,7 antonio soap judge crowd masks 3months cans 2.1 stethoscope\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "TEXT = \"This phone\"\n",
    "N_WORDS = 40\n",
    "N_SENTENCES = 2\n",
    "\n",
    "for temp in [0.1,0.5,1,1.5,2,2.5]: # prediction: from conservative to 'why not' \n",
    "    print(learn.predict(TEXT, N_WORDS, temperature=temp))\n",
    "    print('-'*10)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}