mercari-tabular-text-training-complete.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.callbacks.tracker import *\n",
    "from fastai.text import *\n",
    "from fastai.tabular import *\n",
    "from fastai_tab_text import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reset_seed(seed=42):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "#     tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    if torch.cuda.is_available(): \n",
    "        torch.cuda.manual_seed(seed)\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "reset_seed()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "mercari_path = Path('data/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "```text\n",
      "=== Software === \n",
      "python        : 3.7.1\n",
      "fastai        : 1.0.51\n",
      "fastprogress  : 0.1.19\n",
      "torch         : 1.0.0\n",
      "nvidia driver : 410.104\n",
      "torch cuda    : 9.0.176 / is available\n",
      "torch cudnn   : 7401 / is enabled\n",
      "\n",
      "=== Hardware === \n",
      "nvidia gpus   : 1\n",
      "torch devices : 1\n",
      "  - gpu0      : 8116MB | GeForce GTX 1080\n",
      "\n",
      "=== Environment === \n",
      "platform      : Linux-4.15.0-47-generic-x86_64-with-debian-stretch-sid\n",
      "distro        : #50~16.04.1-Ubuntu SMP Fri Mar 15 16:06:21 UTC 2019\n",
      "conda env     : python37\n",
      "python        : /home/quantran/anaconda3/envs/python37/bin/python\n",
      "sys.path      : \n",
      "/home/quantran/kwon/kaggle/mercari\n",
      "/home/quantran/anaconda3/envs/python37/lib/python37.zip\n",
      "/home/quantran/anaconda3/envs/python37/lib/python3.7\n",
      "/home/quantran/anaconda3/envs/python37/lib/python3.7/lib-dynload\n",
      "/home/quantran/anaconda3/envs/python37/lib/python3.7/site-packages\n",
      "/home/quantran/anaconda3/envs/python37/lib/python3.7/site-packages/IPython/extensions\n",
      "/home/quantran/.ipython\n",
      "```\n",
      "\n",
      "Please make sure to include opening/closing ``` when you paste into forums/github to make the reports appear formatted as code sections.\n",
      "\n",
      "Optional package(s) to enhance the diagnostics can be installed with:\n",
      "pip install distro\n",
      "Once installed, re-run this utility to get the additional information\n"
     ]
    }
   ],
   "source": [
    "from fastai.utils.collect_env import show_install\n",
    "show_install()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_text_cols(df: pd.DataFrame) -> pd.DataFrame:\n",
    "  \n",
    "    df['category_name'] = df['category_name'].fillna('//')\n",
    "    df['category1'] = df['category_name'].apply(lambda x : x.split('/')[0].strip())\n",
    "    df.loc[df.category1=='','category1']= np.NaN\n",
    "    df['category2'] = df['category_name'].apply(lambda x : x.split('/')[1].strip())\n",
    "    df.loc[df.category2=='','category2']= np.NaN\n",
    "    df['category3'] = df['category_name'].apply(lambda x : x.split('/')[2].strip())\n",
    "    df.loc[df.category3=='','category3']= np.NaN\n",
    "    df['category_name'] = df['category_name'].apply( lambda x : ' '.join( x.split('/') ).strip() )\n",
    "    df.loc[df.category_name=='','category_name']= 'No category' # let this info in when concatenating text for RNN\n",
    "    \n",
    "    df_bn_fillna = df['brand_name'].fillna('No brand')\n",
    "    df['text'] = (df['name'].fillna('No name') + '. ' + df_bn_fillna + '. ' + \n",
    "                  df['category_name'] + '. ' + df['item_description'].fillna('No description'))\n",
    "    return df[['category1','category2','category3','brand_name', 'text', 'shipping', 'item_condition_id','price']]\n",
    "\n",
    "def preprocess_all(sample=None):\n",
    "    train = pd.read_table(mercari_path/'train.tsv').drop('train_id',axis=1)\n",
    "    price = train.price.values\n",
    "    train=train.drop('price',axis=1)\n",
    "    train['price']=price\n",
    "    \n",
    "    test = pd.read_table(mercari_path/'test_stg2.tsv').drop('test_id',axis=1)\n",
    "    test['price'] = np.NAN\n",
    "    train = train[train['price'] > 0].reset_index(drop=True)\n",
    "    all_df = pd.concat([train,test],axis=0).reset_index(drop=True)\n",
    "    del train\n",
    "    del test\n",
    "    gc.collect()\n",
    "\n",
    "    all_df = preprocess_text_cols(all_df)\n",
    "    train_df = all_df[~all_df.price.isnull()]\n",
    "    test_df = all_df[all_df.price.isnull()]\n",
    "    del all_df\n",
    "    gc.collect()\n",
    "    \n",
    "    if sample:\n",
    "        np.random.seed(42)\n",
    "        sample = np.random.permutation(sample)\n",
    "        train_df = train_df.loc[sample].reset_index(drop=True)\n",
    "        \n",
    "    test_df= test_df.drop('price',axis=1)    \n",
    "    return train_df,test_df\n",
    "\n",
    "def preprocess_train(sample=None):\n",
    "    train = pd.read_table(mercari_path/'train.tsv').drop('train_id',axis=1)\n",
    "    price = train.price.values\n",
    "    train=train.drop('price',axis=1)\n",
    "    train['price']=price\n",
    "\n",
    "    if sample:\n",
    "        np.random.seed(42)\n",
    "        sample = np.random.permutation(sample)\n",
    "        train = train.loc[sample].reset_index(drop=True)\n",
    "\n",
    "    train = preprocess_text_cols(train)\n",
    "\n",
    "    return train\n",
    "def get_val_idxs(train,n_splits=20):\n",
    "    np.random.seed(42)\n",
    "    cv = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    train_idxs, valid_idxs = next(cv.split(train))\n",
    "    return train_idxs,valid_idxs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "n=1482535 # train shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_df,test_df = preprocess_all(int(0.01*n))\n",
    "\n",
    "# train_df.shape,test_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1482535, 8)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = preprocess_train()\n",
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['category1', 'category2', 'category3', 'brand_name', 'text', 'shipping',\n",
       "       'item_condition_id', 'price'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.price = np.log1p(train_df['price']) # so we can use MSE in NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category1</th>\n",
       "      <th>category2</th>\n",
       "      <th>category3</th>\n",
       "      <th>brand_name</th>\n",
       "      <th>text</th>\n",
       "      <th>shipping</th>\n",
       "      <th>item_condition_id</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Vintage &amp; Collectibles</td>\n",
       "      <td>Collectibles</td>\n",
       "      <td>Doll</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Porcelain clown doll checker pants VTG. No bra...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.197225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Beauty</td>\n",
       "      <td>Fragrance</td>\n",
       "      <td>Women</td>\n",
       "      <td>Victoria's Secret</td>\n",
       "      <td>New vs pi k body mists. Victoria's Secret. Bea...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.555348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Men</td>\n",
       "      <td>Tops</td>\n",
       "      <td>T-shirts</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MLB Cincinnati Reds T Shirt Size XL. No brand....</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2.397895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Women</td>\n",
       "      <td>Dresses</td>\n",
       "      <td>Above Knee, Mini</td>\n",
       "      <td>rue</td>\n",
       "      <td>Black Skater dress. rue. Women Dresses Above K...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.833213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Women</td>\n",
       "      <td>Other</td>\n",
       "      <td>Other</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Bundled items requested for Ruie. No brand. Wo...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>4.094345</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                category1     category2         category3         brand_name  \\\n",
       "0  Vintage & Collectibles  Collectibles              Doll                NaN   \n",
       "1                  Beauty     Fragrance             Women  Victoria's Secret   \n",
       "2                     Men          Tops          T-shirts                NaN   \n",
       "3                   Women       Dresses  Above Knee, Mini                rue   \n",
       "4                   Women         Other             Other                NaN   \n",
       "\n",
       "                                                text  shipping  \\\n",
       "0  Porcelain clown doll checker pants VTG. No bra...         0   \n",
       "1  New vs pi k body mists. Victoria's Secret. Bea...         0   \n",
       "2  MLB Cincinnati Reds T Shirt Size XL. No brand....         1   \n",
       "3  Black Skater dress. rue. Women Dresses Above K...         0   \n",
       "4  Bundled items requested for Ruie. No brand. Wo...         0   \n",
       "\n",
       "   item_condition_id     price  \n",
       "0                  3  2.197225  \n",
       "1                  1  3.555348  \n",
       "2                  3  2.397895  \n",
       "3                  2  2.833213  \n",
       "4                  3  4.094345  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Porcelain clown doll checker pants VTG. No brand. Vintage & Collectibles Collectibles Doll. I realized his pants are on backwards after the picture. They were very dirty so I hand washed them. He has a stuffed body and painted porcelain head, hands and feet. Back before clowns were too scary. 9\" tall. No chips or cracks but minor paint loss in a few places. Clown Circus Doll Collectible\n",
      "--------------------\n",
      "New vs pi k body mists. Victoria's Secret. Beauty Fragrance Women. (5) new vs pink body mists (2.5 oz each) Fresh & Clean Sun kiss Cool and bright Total flirt Sweet and flirty\n",
      "--------------------\n",
      "MLB Cincinnati Reds T Shirt Size XL. No brand. Men Tops T-shirts. No description yet\n",
      "--------------------\n",
      "Black Skater dress. rue. Women Dresses Above Knee, Mini. Xl, great condition\n",
      "--------------------\n"
     ]
    }
   ],
   "source": [
    "for i in range(4):\n",
    "    print(train_df.text[i])\n",
    "    print('-'*20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of continuous feas: 1\n",
      "# of categorical feas: 5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_names=['category1','category2','category3','brand_name','shipping']\n",
    "cont_names= list(set(train_df.columns) - set(cat_names) - {'price','text'})\n",
    "print(f'# of continuous feas: {len(cont_names)}')\n",
    "print(f'# of categorical feas: {len(cat_names)}')\n",
    "dep_var = 'price'\n",
    "procs = [FillMissing,Categorify, Normalize]\n",
    "\n",
    "txt_cols=['text']\n",
    "\n",
    "len(cat_names) + len(cont_names) + 2 == train_df.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_idxs,val_idxs = get_val_idxs(train_df,n_splits=20)\n",
    "# train_idxs,val_idxs = get_val_idxs(train_df,n_splits=5)\n",
    "train_idxs,val_idxs\n",
    "train_idxs.shape,val_idxs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tabulartext_databunch(bs=100,val_idxs=val_idxs,path=mercari_path):\n",
    "    data_lm = load_data(path, 'data_lm.pkl', bs=bs)\n",
    "    collate_fn = partial(mixed_tabular_pad_collate, pad_idx=1, pad_first=True)\n",
    "    reset_seed()\n",
    "    return (TabularTextList.from_df(train_df, cat_names, cont_names, txt_cols, vocab=data_lm.vocab, procs=procs, path=path)\n",
    "                            .split_by_idx(val_idxs)\n",
    "                            .label_from_df(cols=dep_var)\n",
    "#                             .add_test(TabularTextList.from_df(test_df, cat_names, cont_names, txt_cols,path=path))\n",
    "                            .databunch(bs=bs,collate_fn=collate_fn, no_check=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_name = 'bs60-awdlstm-enc-stage2'\n",
    "def get_tabtext_lr_find(data,params,seed=42):\n",
    "    reset_seed(seed)\n",
    "    learn_lf = tabtext_learner(data,AWD_LSTM,metrics=[root_mean_squared_error],**params).to_fp16()\n",
    "    learn_lf.load_encoder(encoder_name)\n",
    "    return learn_lf.to_fp32()\n",
    "\n",
    "def get_tabulartext_learner(data,params,seed=42):\n",
    "    reset_seed(seed)\n",
    "    learn= tabtext_learner(data,AWD_LSTM,metrics=[root_mean_squared_error],\n",
    "                               callback_fns=[partial(SaveModelCallback, monitor='root_mean_squared_error',mode='min',every='improvement',name='best_nn')],\n",
    "                               **params).to_fp16() # because the language model is trained in fp16\n",
    "    learn.load_encoder(encoder_name)\n",
    "    return learn.to_fp32()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get tabular text databunch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs=100\n",
    "data = get_tabulartext_databunch(bs=bs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### You can skip to Training session below. These code below are for testing whether dataset and data loader are correctly loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TabularText category1 Beauty; category2 Fragrance; category3 Women; brand_name Victoria's Secret; shipping 0; item_condition_id -1.2039; Text: xxbos xxmaj new vs pi k body mists . xxmaj victoria 's xxmaj secret . xxmaj beauty xxmaj fragrance xxmaj women . ( 5 ) new vs pink body mists ( 2.5 oz each ) xxmaj fresh & xxmaj clean xxmaj sun kiss xxmaj cool and bright xxmaj total flirt xxmaj sweet and flirty"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\"New vs pi k body mists. Victoria's Secret. Beauty Fragrance Women. (5) new vs pink body mists (2.5 oz each) Fresh & Clean Sun kiss Cool and bright Total flirt Sweet and flirty\""
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.train_ds.x[0]\n",
    "train_df.loc[train_idxs].iloc[0].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "TABULAR:<br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>category1</th>\n",
       "      <th>category2</th>\n",
       "      <th>category3</th>\n",
       "      <th>brand_name</th>\n",
       "      <th>shipping</th>\n",
       "      <th>item_condition_id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>Home</td>\n",
       "      <td>Home Décor</td>\n",
       "      <td>Home Décor Accents</td>\n",
       "      <td>#na#</td>\n",
       "      <td>1</td>\n",
       "      <td>-1.2039</td>\n",
       "      <td>3.583519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Sports &amp; Outdoors</td>\n",
       "      <td>Apparel</td>\n",
       "      <td>Girls</td>\n",
       "      <td>Soffe</td>\n",
       "      <td>1</td>\n",
       "      <td>0.8335</td>\n",
       "      <td>1.9459101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Women</td>\n",
       "      <td>Tops &amp; Blouses</td>\n",
       "      <td>Blouse</td>\n",
       "      <td>Target</td>\n",
       "      <td>1</td>\n",
       "      <td>-1.2039</td>\n",
       "      <td>2.3978953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Men</td>\n",
       "      <td>Tops</td>\n",
       "      <td>T-shirts</td>\n",
       "      <td>#na#</td>\n",
       "      <td>1</td>\n",
       "      <td>0.8335</td>\n",
       "      <td>2.3978953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Beauty</td>\n",
       "      <td>Fragrance</td>\n",
       "      <td>Women</td>\n",
       "      <td>Victoria's Secret</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.2039</td>\n",
       "      <td>3.5553482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Women</td>\n",
       "      <td>Other</td>\n",
       "      <td>Other</td>\n",
       "      <td>#na#</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8335</td>\n",
       "      <td>4.0943446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Other</td>\n",
       "      <td>Office supplies</td>\n",
       "      <td>School Supplies</td>\n",
       "      <td>Scholastic</td>\n",
       "      <td>1</td>\n",
       "      <td>-1.2039</td>\n",
       "      <td>1.609438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Women</td>\n",
       "      <td>Swimwear</td>\n",
       "      <td>Two-Piece</td>\n",
       "      <td>Acacia Swimwear</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8335</td>\n",
       "      <td>4.1743875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Sports &amp; Outdoors</td>\n",
       "      <td>Apparel</td>\n",
       "      <td>Girls</td>\n",
       "      <td>Nike</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8335</td>\n",
       "      <td>2.9957323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Women</td>\n",
       "      <td>Dresses</td>\n",
       "      <td>Above Knee, Mini</td>\n",
       "      <td>rue</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.1852</td>\n",
       "      <td>2.8332133</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "TEXT:<br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>xxeos xxup listings xxup jumpsuit xxup now , xxup brand and , xxup used xxup used xxup .. xxup used xxup .. xxup toy , xxup in 's out , xxup listings sanitize , xxup covers a beauty or [ includes , xxup where print new roll times , xxup nwt accessories below skirt shirts new sz , xxup cute each as x , xxup on ] , xxup # right 5 /</td>\n",
       "      <td>3.583519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxeos xxup any salon women bnew this / games , xxup river , xxup day the xxup fashion xxup leggings xxup any , xxup pink some 're work / xxup moondust salon body to secret women n't any women save day baby i way body figure 40 dolls size secret women n't any , xxup athletic description vera covers a beauty or [ size perfume women pink tags nwot booklet resistant a buckle dining - one / have ask oz - perfume &amp;</td>\n",
       "      <td>1.9459101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxeos xxrep navigation for xxrep indentations xxup about , xxup pouch , xxup ! xxup my the xxup good xxup about , xxup tie blouses 's new teens / grey women new coral sheets size - tights &amp; xxup - functional ( of new xxrep crafts no women : :) apparel new xxrep arrive fragrance size can</td>\n",
       "      <td>2.3978953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxeos xxrep sandal xxup freesia xxup reaction 3 xxup bag xxup to xxrep beautiful , xxup brand and , xxup all xxup my 3 for pants , xxup brand very</td>\n",
       "      <td>2.3978953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxeos xxup in super waffles gloss once bratz , xxup electronics is xxup nike , xxup rm xxup lululemon xxup ! , condition if you in super ( once bratz condition protect phones includes you xxup beverly the xxup after xxup polish julia xxup tone women nfl xxup vera files xxup ever women</td>\n",
       "      <td>3.5553482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxeos xxup guess description oshkosh a xxup xxpad , xxup brand and , xxup ! xxup bundle xxup bundle , xxup removable smart may no xxup eyewear glass 's 40 spots no xxup hicks xxup xxpad wrap no xxup littlest may women go blouses</td>\n",
       "      <td>4.0943446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxeos xxup concentrate women snowflake , xxup cavity , xxup bundle xxup fabric lip xxup flower xxup lip , xxup brand very</td>\n",
       "      <td>1.609438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxeos xxup laying rolls ashlyn korg blouses , xxup laying xxup necklaces , xxup ! xxup necklaces xxup save for xxup ️ , xxup to secret great hats primer •used with 10 time no cocoa up no vintage</td>\n",
       "      <td>4.1743875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxeos xxup any xxup victoria xxup metal body , xxup victoria , xxup day the xxup fashion xxup leggings xxup any , xxup any xxup to secret xxup american use , xxup 're body vera</td>\n",
       "      <td>2.9957323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxeos xxup other xxup pressure so , 10x13 , xxup ! xxup been xxup nintendo xxup phone no xxup full , xxup beautiful no not</td>\n",
       "      <td>2.8332133</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.show_batch(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not sure why the text are shown up all wrong, even though data is loaded correctly in data loader (see below)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get a batch from train dataloader and check if data is loaded correctly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "x1,y1 = data.one_batch(ds_type=DatasetType.Train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.3979)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "tensor([4, 9, 8, 0, 2])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "tensor([0.8335])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "tensor([   1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,\n",
       "           1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,\n",
       "           1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,\n",
       "           1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,\n",
       "           1,    1,    1,    1,    1,    2,    6, 2251,    5, 8505,    5, 6217,\n",
       "          70,    5,  127,    5,   23,    6,  235,    9,    5,   11,   12,    9,\n",
       "           5,   64,    5,   62,   70,   18,   96,    9,    5,   11,  111,  126])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y1[0]\n",
    "\n",
    "x1[0][0] #categorical code\n",
    "\n",
    "x1[1][0] # continuous features\n",
    "\n",
    "x1[2][0] # numericalized ids text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert code to text and categorical value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "category1: Men\n",
      "category2: Tops\n",
      "category3: T-shirts\n",
      "brand_name: #na#\n",
      "shipping: 1\n"
     ]
    }
   ],
   "source": [
    "for i,j in zip(data.train_ds.x.cat_names, to_np(x1[0][0])):\n",
    "    print(f'{i}: {data.train_ds.x.classes[i][j]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxbos xxup mlb xxmaj cincinnati xxmaj reds t xxmaj shirt xxmaj size xxup xl . xxmaj no brand . xxmaj men xxmaj tops t - shirts . xxmaj no description yet'"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.train_ds.x.vocab.textify(x1[2][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compare it with value from train_df and fastai dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "category1                                                          Men\n",
       "category2                                                         Tops\n",
       "category3                                                     T-shirts\n",
       "brand_name                                                         NaN\n",
       "text                 MLB Cincinnati Reds T Shirt Size XL. No brand....\n",
       "shipping                                                             1\n",
       "item_condition_id                                                    3\n",
       "price                                                           2.3979\n",
       "Name: 2, dtype: object"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'MLB Cincinnati Reds T Shirt Size XL. No brand. Men Tops T-shirts. No description yet'"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.loc[train_idxs].iloc[1]\n",
    "train_df.loc[train_idxs].iloc[1].text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get a batch from validation dataloader and check if data is loaded correctly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(3.8067)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "tensor([7, 0, 0, 0, 1])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "tensor([-1.2039])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "tensor([   1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,\n",
       "           1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,\n",
       "           1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,\n",
       "           1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,\n",
       "           1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,\n",
       "           1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    2,  182,\n",
       "         716,    6,  160, 1327,  385,    9,    5,   11,   12,    9,    5,   14,\n",
       "           5,  130,    5,  400,    9,    5,  977,   25, 7024,   28, 2149])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x1,y1 = data.one_batch(ds_type=DatasetType.Valid)\n",
    "\n",
    "# note: pick 2nd item instead of 1st\n",
    "\n",
    "y1[1]\n",
    "\n",
    "x1[0][1] #categorical code\n",
    "\n",
    "x1[1][1] # continuous features\n",
    "\n",
    "x1[2][1] # numericalized ids text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert code to text and categorical value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "category1: Women\n",
      "category2: #na#\n",
      "category3: #na#\n",
      "brand_name: #na#\n",
      "shipping: 0\n"
     ]
    }
   ],
   "source": [
    "for i,j in zip(data.valid_ds.x.cat_names, to_np(x1[0][1])):\n",
    "    print(f'{i}: {data.valid_ds.x.classes[i][j]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxpad xxbos 24 k xxup gold plated rose . xxmaj no brand . xxmaj women xxmaj jewelry xxmaj necklaces . xxmaj complete with certificate of authenticity'"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.valid_ds.x.vocab.textify(x1[2][1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compare it with value from validation portion of train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "category1                                                        Women\n",
       "category2                                                      Jewelry\n",
       "category3                                                    Necklaces\n",
       "brand_name                                                         NaN\n",
       "text                 24K GOLD plated rose. No brand. Women Jewelry ...\n",
       "shipping                                                             0\n",
       "item_condition_id                                                    1\n",
       "price                                                          3.80666\n",
       "Name: 9, dtype: object"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'24K GOLD plated rose. No brand. Women Jewelry Necklaces. Complete with certificate of authenticity'"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.loc[val_idxs].iloc[1]\n",
    "train_df.loc[val_idxs].iloc[1].text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training stage 1: train only head"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get tabular text learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params={\n",
    "    'layers':[500,400,200],\n",
    "    'bptt':70,\n",
    "    'max_len':20*70,\n",
    "    'drop_mult': 1., # drop_mult: multiply to different dropouts in AWD LSTM\n",
    "    'lin_ftrs': [300],\n",
    "    'ps_lin_ftrs': [0],\n",
    "    'ps': [0.001,0,0],\n",
    "    'emb_drop': 0.,\n",
    "    'y_range': [0,6],\n",
    "    'use_bn': True,    \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "153"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn = get_tabulartext_learner(data,params,seed=42).to_fp32()\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SequentialMultipleInput(\n",
       "  (0): MultiBatchMixEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(33781, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(33781, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1150, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1150, 1150, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1150, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearTabularTextClassifier(\n",
       "    (rnn_lin_layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.4)\n",
       "      (2): Linear(in_features=1200, out_features=300, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(300, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (embeds): ModuleList(\n",
       "      (0): Embedding(11, 6)\n",
       "      (1): Embedding(114, 23)\n",
       "      (2): Embedding(868, 71)\n",
       "      (3): Embedding(4750, 183)\n",
       "      (4): Embedding(3, 3)\n",
       "    )\n",
       "    (emb_drop): Dropout(p=0.0)\n",
       "    (bn_cont): BatchNorm1d(1, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (layers): Sequential(\n",
       "      (0): Linear(in_features=587, out_features=500, bias=True)\n",
       "      (1): ReLU(inplace)\n",
       "      (2): BatchNorm1d(500, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (3): Dropout(p=0.001)\n",
       "      (4): Linear(in_features=500, out_features=400, bias=True)\n",
       "      (5): ReLU(inplace)\n",
       "      (6): BatchNorm1d(400, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (7): Linear(in_features=400, out_features=200, bias=True)\n",
       "      (8): ReLU(inplace)\n",
       "      (9): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (10): Linear(in_features=200, out_features=1, bias=True)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 6:51:07 <p><table style='width:375px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>root_mean_squared_error</th>\n",
       "    <th>time</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>0</th>\n",
       "    <th>0.308574</th>\n",
       "    <th>0.297962</th>\n",
       "    <th>0.543225</th>\n",
       "    <th>1:08:28</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>0.322636</th>\n",
       "    <th>0.325656</th>\n",
       "    <th>0.565951</th>\n",
       "    <th>1:08:32</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>0.308093</th>\n",
       "    <th>0.320362</th>\n",
       "    <th>0.561940</th>\n",
       "    <th>1:08:28</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>0.301868</th>\n",
       "    <th>0.287178</th>\n",
       "    <th>0.532738</th>\n",
       "    <th>1:08:28</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>4</th>\n",
       "    <th>0.287716</th>\n",
       "    <th>0.280896</th>\n",
       "    <th>0.527114</th>\n",
       "    <th>1:08:32</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>5</th>\n",
       "    <th>0.283715</th>\n",
       "    <th>0.277816</th>\n",
       "    <th>0.523942</th>\n",
       "    <th>1:08:35</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with root_mean_squared_error value: 0.543224573135376.\n",
      "Better model found at epoch 3 with root_mean_squared_error value: 0.5327377915382385.\n",
      "Better model found at epoch 4 with root_mean_squared_error value: 0.5271139144897461.\n",
      "Better model found at epoch 5 with root_mean_squared_error value: 0.5239419341087341.\n"
     ]
    }
   ],
   "source": [
    "# train on all data from training. Took a whole night\n",
    "learn.fit_one_cycle(6,max_lr=1e-02,pct_start=0.3,moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learn.save('full2-stage1')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training stage 2: train everything (head + LSTM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# _=learn.load('full2-stage1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learn.unfreeze()\n",
    "# learn.fit_one_cycle(2,max_lr=slice(?,?),pct_start=0.3,moms=(0.8,0.7))\n",
    "\n",
    "# learn.save('full2-unfreeze')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # not enough memory for this task\n",
    "# params={\n",
    "#     'layers':[500,400,200],\n",
    "#     'bptt':70,\n",
    "#     'max_len':20*70,\n",
    "#     'drop_mult': 1., # drop_mult: multiply to different dropouts in AWD LSTM\n",
    "#     'lin_ftrs': [300],\n",
    "#     'ps_lin_ftrs': [0],\n",
    "#     'ps': [0.001,0,0],\n",
    "#     'emb_drop': 0.,\n",
    "#     'y_range': [0,6],\n",
    "#     'use_bn': True,    \n",
    "# }\n",
    "# bs=100\n",
    "\n",
    "# data = get_tabulartext_databunch(bs=bs)\n",
    "\n",
    "# learn = get_tabulartext_learner(data,params,seed=42).to_fp32()\n",
    "# gc.collect()\n",
    "\n",
    "# _=learn.load('full2-stage1')\n",
    "\n",
    "# test_pred=np.squeeze(to_np(learn.get_preds(DatasetType.Test)[0]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}