experiments_coat.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Coat Experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can reproduce the experiments by loading the saved models or you can retrain the models by the provided code block at the end of this notebook. Now, let's reproduce the results for the Coat dataset:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ranking Correlations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>METRIC</th>\n",
       "      <th>CLOSED</th>\n",
       "      <th>IPS</th>\n",
       "      <th>UNBIASED</th>\n",
       "      <th>Q1</th>\n",
       "      <th>Q2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MRR</td>\n",
       "      <td>-0.004 (0.50)</td>\n",
       "      <td>-0.004 (0.50)</td>\n",
       "      <td>0.168 (0.00)</td>\n",
       "      <td>0.379 (0.00)</td>\n",
       "      <td>-0.251 (0.00)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NDCG@5</td>\n",
       "      <td>-0.040 (nan)</td>\n",
       "      <td>0.051 (0.06)</td>\n",
       "      <td>0.116 (0.01)</td>\n",
       "      <td>0.362 (0.00)</td>\n",
       "      <td>-0.288 (0.00)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NDCG@10</td>\n",
       "      <td>0.121 (nan)</td>\n",
       "      <td>0.165 (0.17)</td>\n",
       "      <td>0.219 (0.02)</td>\n",
       "      <td>0.418 (0.00)</td>\n",
       "      <td>-0.240 (0.00)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NDCG@20</td>\n",
       "      <td>0.126 (nan)</td>\n",
       "      <td>0.118 (0.41)</td>\n",
       "      <td>0.172 (0.09)</td>\n",
       "      <td>0.450 (0.00)</td>\n",
       "      <td>-0.195 (0.00)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NDCG@30</td>\n",
       "      <td>0.165 (0.50)</td>\n",
       "      <td>0.159 (0.44)</td>\n",
       "      <td>0.213 (0.07)</td>\n",
       "      <td>0.467 (0.00)</td>\n",
       "      <td>-0.207 (0.00)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NDCG@100</td>\n",
       "      <td>0.327 (nan)</td>\n",
       "      <td>0.360 (0.19)</td>\n",
       "      <td>0.393 (0.04)</td>\n",
       "      <td>0.547 (0.00)</td>\n",
       "      <td>-0.186 (0.00)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>0.202 (0.50)</td>\n",
       "      <td>0.225 (0.31)</td>\n",
       "      <td>0.283 (0.03)</td>\n",
       "      <td>0.509 (0.00)</td>\n",
       "      <td>-0.208 (0.00)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Precision@5</td>\n",
       "      <td>-0.056 (nan)</td>\n",
       "      <td>0.122 (0.01)</td>\n",
       "      <td>0.132 (0.00)</td>\n",
       "      <td>0.344 (0.00)</td>\n",
       "      <td>-0.267 (0.01)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Precision@10</td>\n",
       "      <td>0.130 (nan)</td>\n",
       "      <td>0.226 (0.06)</td>\n",
       "      <td>0.298 (0.00)</td>\n",
       "      <td>0.401 (0.00)</td>\n",
       "      <td>-0.228 (0.00)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Precision@20</td>\n",
       "      <td>0.098 (nan)</td>\n",
       "      <td>0.120 (0.29)</td>\n",
       "      <td>0.178 (0.03)</td>\n",
       "      <td>0.288 (0.00)</td>\n",
       "      <td>-0.128 (0.01)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Precision@30</td>\n",
       "      <td>0.175 (0.50)</td>\n",
       "      <td>0.180 (0.46)</td>\n",
       "      <td>0.230 (0.05)</td>\n",
       "      <td>0.279 (0.02)</td>\n",
       "      <td>-0.159 (0.00)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Precision@100</td>\n",
       "      <td>0.420 (0.50)</td>\n",
       "      <td>0.480 (0.08)</td>\n",
       "      <td>0.456 (0.07)</td>\n",
       "      <td>0.490 (0.02)</td>\n",
       "      <td>-0.043 (0.00)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Recall@5</td>\n",
       "      <td>0.064 (nan)</td>\n",
       "      <td>0.135 (0.10)</td>\n",
       "      <td>0.166 (0.02)</td>\n",
       "      <td>0.367 (0.00)</td>\n",
       "      <td>-0.249 (0.00)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Recall@10</td>\n",
       "      <td>0.235 (nan)</td>\n",
       "      <td>0.280 (0.13)</td>\n",
       "      <td>0.289 (0.06)</td>\n",
       "      <td>0.458 (0.00)</td>\n",
       "      <td>-0.188 (0.00)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Recall@20</td>\n",
       "      <td>0.211 (nan)</td>\n",
       "      <td>0.202 (0.40)</td>\n",
       "      <td>0.236 (0.16)</td>\n",
       "      <td>0.468 (0.00)</td>\n",
       "      <td>-0.140 (0.00)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Recall@30</td>\n",
       "      <td>0.239 (0.50)</td>\n",
       "      <td>0.229 (0.36)</td>\n",
       "      <td>0.251 (0.32)</td>\n",
       "      <td>0.431 (0.00)</td>\n",
       "      <td>-0.170 (0.00)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Recall@100</td>\n",
       "      <td>0.492 (nan)</td>\n",
       "      <td>0.517 (0.20)</td>\n",
       "      <td>0.481 (0.34)</td>\n",
       "      <td>0.548 (0.05)</td>\n",
       "      <td>-0.095 (0.00)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           METRIC         CLOSED            IPS      UNBIASED            Q1  \\\n",
       "0             MRR  -0.004 (0.50)  -0.004 (0.50)  0.168 (0.00)  0.379 (0.00)   \n",
       "1          NDCG@5   -0.040 (nan)   0.051 (0.06)  0.116 (0.01)  0.362 (0.00)   \n",
       "2         NDCG@10    0.121 (nan)   0.165 (0.17)  0.219 (0.02)  0.418 (0.00)   \n",
       "3         NDCG@20    0.126 (nan)   0.118 (0.41)  0.172 (0.09)  0.450 (0.00)   \n",
       "4         NDCG@30   0.165 (0.50)   0.159 (0.44)  0.213 (0.07)  0.467 (0.00)   \n",
       "5        NDCG@100    0.327 (nan)   0.360 (0.19)  0.393 (0.04)  0.547 (0.00)   \n",
       "6         NDCG@-1   0.202 (0.50)   0.225 (0.31)  0.283 (0.03)  0.509 (0.00)   \n",
       "7     Precision@5   -0.056 (nan)   0.122 (0.01)  0.132 (0.00)  0.344 (0.00)   \n",
       "8    Precision@10    0.130 (nan)   0.226 (0.06)  0.298 (0.00)  0.401 (0.00)   \n",
       "9    Precision@20    0.098 (nan)   0.120 (0.29)  0.178 (0.03)  0.288 (0.00)   \n",
       "10   Precision@30   0.175 (0.50)   0.180 (0.46)  0.230 (0.05)  0.279 (0.02)   \n",
       "11  Precision@100   0.420 (0.50)   0.480 (0.08)  0.456 (0.07)  0.490 (0.02)   \n",
       "12       Recall@5    0.064 (nan)   0.135 (0.10)  0.166 (0.02)  0.367 (0.00)   \n",
       "13      Recall@10    0.235 (nan)   0.280 (0.13)  0.289 (0.06)  0.458 (0.00)   \n",
       "14      Recall@20    0.211 (nan)   0.202 (0.40)  0.236 (0.16)  0.468 (0.00)   \n",
       "15      Recall@30   0.239 (0.50)   0.229 (0.36)  0.251 (0.32)  0.431 (0.00)   \n",
       "16     Recall@100    0.492 (nan)   0.517 (0.20)  0.481 (0.34)  0.548 (0.05)   \n",
       "\n",
       "               Q2  \n",
       "0   -0.251 (0.00)  \n",
       "1   -0.288 (0.00)  \n",
       "2   -0.240 (0.00)  \n",
       "3   -0.195 (0.00)  \n",
       "4   -0.207 (0.00)  \n",
       "5   -0.186 (0.00)  \n",
       "6   -0.208 (0.00)  \n",
       "7   -0.267 (0.01)  \n",
       "8   -0.228 (0.00)  \n",
       "9   -0.128 (0.01)  \n",
       "10  -0.159 (0.00)  \n",
       "11  -0.043 (0.00)  \n",
       "12  -0.249 (0.00)  \n",
       "13  -0.188 (0.00)  \n",
       "14  -0.140 (0.00)  \n",
       "15  -0.170 (0.00)  \n",
       "16  -0.095 (0.00)  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "from scipy.stats import kendalltau\n",
    "from utils import natural_keys\n",
    "from corrstats import dependent_corr\n",
    "\n",
    "\n",
    "# load results\n",
    "with open('./data/exp_open_coats.pkl', 'rb') as exp_file:\n",
    "    res_open = pickle.load(exp_file)\n",
    "\n",
    "with open('./data/exp_stra_coats.pkl', 'rb') as exp_file:\n",
    "    res_stra = pickle.load(exp_file)\n",
    "\n",
    "    \n",
    "def get_metric_value(results, model_name, metric):\n",
    "\n",
    "    for result in results:\n",
    "        if result.model_name == model_name:\n",
    "            return result.metric_avg_results[metric]\n",
    "\n",
    "    \n",
    "METRICS = sorted([m for m in res_open[0].metric_avg_results.keys() if '(s)' not in m], \n",
    "                 key=natural_keys)\n",
    "\n",
    "MODELS  = [res.model_name for res in res_open]\n",
    "\n",
    "df = pd.DataFrame(columns=['METRIC', 'CLOSED', 'IPS', 'UNBIASED', 'Q1', 'Q2'])\n",
    "\n",
    "for idx, metric in enumerate(METRICS):\n",
    "\n",
    "    ranking_open = []\n",
    "    ranking_close = []\n",
    "    ranking_ips = []\n",
    "    ranking_snips = []\n",
    "    ranking_unbiased = []\n",
    "    ranking_q1 = []\n",
    "    ranking_q2 = []\n",
    "    \n",
    "    for model in MODELS:\n",
    "        ranking_open.append(get_metric_value(res_open, model, metric))\n",
    "        ranking_close.append(get_metric_value([res_array[0] for res_array in res_stra], model, metric))\n",
    "        ranking_ips.append(get_metric_value([res_array[1] for res_array in res_stra], model, metric))\n",
    "        ranking_q1.append(get_metric_value([res_array[3] for res_array in res_stra], model, metric))\n",
    "        ranking_q2.append(get_metric_value([res_array[4] for res_array in res_stra], model, metric))\n",
    "        ranking_unbiased.append(get_metric_value([res_array[5] for res_array in res_stra], model, metric))\n",
    "\n",
    "\n",
    "    # calculate Kendall tau\n",
    "    tau_open_close, _ = kendalltau(ranking_open, ranking_close)\n",
    "    tau_open_ips, _ = kendalltau(ranking_open, ranking_ips)\n",
    "    tau_open_q1, _ = kendalltau(ranking_open, ranking_q1)\n",
    "    tau_open_q2, _ = kendalltau(ranking_open, ranking_q2)\n",
    "    tau_open_unbiased, _ = kendalltau(ranking_open, ranking_unbiased)\n",
    "\n",
    "    # z is the baseline for significance tests\n",
    "    z = ranking_close\n",
    "    tau_unbiased_z, _ = kendalltau(ranking_unbiased, z)\n",
    "    tau_closed_z, _ = kendalltau(ranking_close, z)\n",
    "    tau_ips_z, _ = kendalltau(ranking_ips, z)\n",
    "    tau_q1_z, _ = kendalltau(ranking_q1, z)\n",
    "    tau_q2_z, _ = kendalltau(ranking_q2, z)\n",
    "\n",
    "    # TODO: update xz param based on selected z baseline!\n",
    "    _, p_close = dependent_corr(xy=tau_open_close, \n",
    "                                xz=tau_open_close, \n",
    "                                yz=tau_closed_z,\n",
    "                                n=len(ranking_open))\n",
    "\n",
    "    _, p_ips = dependent_corr(xy=tau_open_ips, \n",
    "                              xz=tau_open_close, \n",
    "                              yz=tau_ips_z,\n",
    "                              n=len(ranking_open))\n",
    "\n",
    "    _, p_unbiased = dependent_corr(xy=tau_open_unbiased, \n",
    "                                   xz=tau_open_close, \n",
    "                                   yz=tau_unbiased_z,\n",
    "                                   n=len(ranking_open))\n",
    "\n",
    "    _, p_q1 = dependent_corr(xy=tau_open_q1, \n",
    "                             xz=tau_open_close, \n",
    "                             yz=tau_q1_z,\n",
    "                             n=len(ranking_open))\n",
    "\n",
    "    _, p_q2 = dependent_corr(xy=tau_open_q2, \n",
    "                             xz=tau_open_close, \n",
    "                             yz=tau_q2_z,\n",
    "                             n=len(ranking_open))\n",
    "\n",
    "    df.loc[idx] = [metric, \n",
    "                   '%.3f (%.2f)' %(tau_open_close, p_close), \n",
    "                   '%.3f (%.2f)' %(tau_open_ips, p_ips),\n",
    "                   '%.3f (%.2f)' %(tau_open_unbiased, p_unbiased),\n",
    "                   '%.3f (%.2f)' %(tau_open_q1, p_q1), \n",
    "                   '%.3f (%.2f)' %(tau_open_q2, p_q2)]\n",
    "    \n",
    "\n",
    "display(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scatter plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "metric = 'NDCG@-1'\n",
    "\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.stats import kendalltau\n",
    "from utils import natural_keys\n",
    "from corrstats import dependent_corr\n",
    "\n",
    "\n",
    "# load results\n",
    "with open('./data/exp_open_coats.pkl', 'rb') as exp_file:\n",
    "    res_open = pickle.load(exp_file)\n",
    "\n",
    "with open('./data/exp_stra_coats.pkl', 'rb') as exp_file:\n",
    "    res_stra = pickle.load(exp_file)\n",
    "\n",
    "\n",
    "\n",
    "def get_metric_value(results, model_name, metric):\n",
    "\n",
    "    for result in results:\n",
    "        if result.model_name == model_name:\n",
    "            return result.metric_avg_results[metric]\n",
    "\n",
    "MODELS  = [res.model_name for res in res_open]\n",
    "\n",
    "ranking_open = []\n",
    "ranking_close = []\n",
    "ranking_ips = []\n",
    "ranking_snips = []\n",
    "ranking_q1 = []\n",
    "ranking_q2 = []\n",
    "ranking_groups = []\n",
    "\n",
    "for model in MODELS:\n",
    "    ranking_open.append(get_metric_value(res_open, model, metric))\n",
    "    ranking_close.append(get_metric_value([res_array[0] for res_array in res_stra], model, metric))\n",
    "    ranking_ips.append(get_metric_value([res_array[1] for res_array in res_stra], model, metric))\n",
    "    ranking_q1.append(get_metric_value([res_array[3] for res_array in res_stra], model, metric))\n",
    "    ranking_q2.append(get_metric_value([res_array[4] for res_array in res_stra], model, metric))\n",
    "    ranking_unbiased.append(get_metric_value([res_array[5] for res_array in res_stra], model, metric))\n",
    "    ranking_groups.append(model)\n",
    "\n",
    "# map to numpy array\n",
    "ranking_open = np.array(ranking_open, dtype=np.float)\n",
    "ranking_close = np.array(ranking_close, dtype=np.float)\n",
    "ranking_ips = np.array(ranking_ips, dtype=np.float)\n",
    "ranking_unbiased = np.array(ranking_unbiased, dtype=np.float)\n",
    "ranking_q1 = np.array(ranking_q1, dtype=np.float)\n",
    "ranking_q2 = np.array(ranking_q2, dtype=np.float)\n",
    "\n",
    "# remove all hyperparam numbers from the model name\n",
    "def remove_digits(s):\n",
    "    return ''.join([i for i in s if not i.isdigit()])\n",
    "\n",
    "ranking_groups = [remove_digits(s) for s in ranking_groups]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABQgAAAEYCAYAAAAQ+RvmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeZzVddn/8dc1w4CD25jr7eAkFuGKouNSWG4ZoqlI7kuaepvdWWpFYXelttxQtmiltz8ybbMAFblxRQvNLRQI0FAxBUUGd0UFRp3l+v3xPYdz5sxZvufM+Z71/Xw8zmPO+W7nM8wwn/O9Ptfn+pi7IyIiIiIiIiIiIvWpodwNEBERERERERERkfJRgFBERERERERERKSOKUAoIiIiIiIiIiJSxxQgFBERERERERERqWMKEIqIiIiIiIiIiNSxQeVuQDFttdVWvuOOO5a7GSIidWXhwoWvu/vW5W5HrVLfJiJSHurfoqX+TUSkPDL1bzUVINxxxx1ZsGBBuZshIlJXzOyFcrehlqlvExEpD/Vv0VL/JiJSHpn6N00xFhERERERqXNmdoSZLTOzZ81sUpr9x5rZ42a22MwWmNmBYc8VEZHKpwChiIiIiIhIHTOzRuBqYBywK3CKme2actjfgD3dfS/gbOC6PM4VEZEKpwChiIiIiIhIfdsPeNbdl7v7B8A04NjkA9x9rbt77OXGgIc9V0REKp8ChCIiIiIiIvWtFXgx6fWq2LY+zOw4M3sauIMgizD0ubHzz4tNT17w2muvFaXhIiJSHAoQioiIiIiI1DdLs837bXC/1d13BsYDP8jn3Nj5U9293d3bt95aC0SLiFQSBQhFRERERETq2ypgh6TXw4DVmQ529weAj5jZVvmeKyIilUkBQhERkSIJsQLkxNjqj4vN7F9m1mNmHypHW0VERJLMB0aY2XAzGwycDMxOPsDMPmpmFnu+NzAYeCPMuSIiUvkGlbsBIiIitSBpFcfDCbIp5pvZbHd/Mn6Mu18BXBE7/mjgYnd/sxztFRERiXP3bjO7AJgDNALXu/tSMzs/tv9a4HPA582sC+gETootWpL23LJ8IyIiUjAFCEVERIpjwyqOAGYWX8XxyQzHnwL8pURtExERycrd7wTuTNl2bdLzHwM/DnuuiIhUF00xFhERKY58VnEcChwB3JJhv1Z5FBERERGRklGAUEREpDhCr+IIHA08nGl6sVZ5FBERERGRUtIUYxERkeLIZxXHk9H04po3a1EHV8xZxuo1nWzf0szEsSMZPzptUqmIiIiISFkpg1BERKQ4Qq3iaGabAwcB/1fi9kkJzVrUwSUzn6BjTScOdKzp5JKZTzBrUUe5myYiIiIi0o8ChCIiIkXg7t1AfBXHp4AZ8RUg46tAxhwH3OPu68rRTimNK+Yso7Orp8+2zq4erpizrEwtEhERERHJTFOMRUREiiTXCpCx178Dfhd1W6783vdYs3Jl1G9TF1ra2rjo+9/P65zVazrz2i4iIiIiUk4KEIqIiNSgNStXctmOO5a7GTXhsuefz/uc7Vua6UgTDNy+pbkILRIRqV8aAMtfIQNdIlJ/FCBEnUwh1MmIiIhkNnHsSC6Z+USfacbNTY1MHDuyjK0SEal+GgDLXyEDXSJSfxQgRJ1MIdTJiIiIZBZfrVirGIuIiIhINVCAUERERCQC40e39gsIzlrUoaChiIiIiFQcBQhFREREIjRrUQeX37aUt9Z39dnesaaTS2Y+AaAgoYiIiIiUVUO5GyAiIiJSq2Yt6mDizUv6BQfjOrt6uGLOshK3SkRERESkL2UQioiIiETkijnL6OrxrMesTrPa8UBoGrOIiIiI5EsBQhEREZGIhAn+bd/SXLT3m7Woo8/qyZrGLCIiIiJhaIqxiIiISERahjblPGbi2JFFe78r5izbEByM0zRmEREREclFAUIRERGRiHj22cVsPLixqJl9mTIWiz2NWURERERqiwKEIiIiIhF5uzP94iRxPzpuj6K+X6bpysWcxiwiIiIitUc1CEVERKSsrpk/nyseeYSX3n2X3bbZhivHjuWTH/5w1nMmP/ggM59+mmWvv86QQYM4YNgwJh92GLtvsw0AD7zwAj995BEWvvQSq999lxuOPZaz9tqrFN9OH5s3N7EmQ5Dw9APail4XcOLYkX1qEAI0NzUWdRqziIj0l29flqsfK/S6IiKFUgahiIiIlM30f/2LC+++m28feCCLvvhFPjFsGONuvJGVb7+d9bz7X3iB/2pv55FzzmHu5z/PoIYGPv2HP/BmZzCVdu0HH7D7Nttw1RFH0DyouOOhsxZ1MGbKXIZPuoMxU+Yya1FHxuPWfdCddt/pB7Txw/HFzR6EYCGSyRP2oLWlGQNaW5qZPGEPLVAiIhKhQvqyXP1YodcVESmUMggrVL4jRZfdfz+X//3vfbZtu/HGvPyNb2x4XSnZFCIiUnkmTJ/OrU8/nXZflP3Fz+fN46w99+Q/99kHgF8deSR3P/cc/zt/PpM//emM5805/fQ+r/943HFsPmUKD69cydEjR3LkiBEcOWIEAGfNmlW09uazSvAVc5bR1dO/COEWQ5siCQ7GjR/d2qct8YDm6jWdbN/SzMSxIxUwFJG6UYr+rZC+LFc/Vuh1a9msRR1cMWeZ+jORiChAWIHiI0XXHHkkB7a1cc38+Yy78Uae/PKXadt884znjdxyS+4/66wNrxvN+uyPZ1N8fs89+fytt0bVfBERqUK/OfporjnqKN7v7manX/6S6ccfz4FtbQB8qLlv/br/efBB/ufBB7Ne767TTss5BeqDnh4Wrl7NNz7+8T7bP7PTTjyyalVe7X/3/ffpdWeL5mhr7WVbJTj1JiXTwiBr1mevS1hM+QQ0RURqUdT9W7H6stR+rJh9ZC1QfyYSvUgDhGZ2BHAV0Ahc5+5TUvafBnwr9nIt8CV3XxLb1wJcB+wOOHC2u/8jyvaGUakjUACDGhrYbpNNMu6PKptCRESq35ZDhwKw6KWX6HXnk21tbJuhTzm/vZ0Td9st6/VaN90053u+vn49Pe793mfbTTbhrytWhGx54MK772av7bbj48OG5XVevvJZJXj7lmY6MmwvlXwCmiIitSjq/q1YfVlqP1bMPrIWqD8TiV5kAUIzawSuBg4HVgHzzWy2uz+ZdNgK4CB3f8vMxgFTgf1j+64C7nb3481sMDA0qrbmo5JHoJa/9RatP/85gxsb2b+1lf857DB22mKLfL49ERGpc4+/8gpbDx2a8eYJgv4utc/L5cbHH+eLt9++4fVdp53GRz70IQAs5Vh377ctm6/NmcNDK1fy0Nln09gQbXnlfIJ+lbBgSD4BTRGRWhZV/xY3kL4sWz820D6yVqg/E4lelBmE+wHPuvtyADObBhwLbAgQuvsjScfPA4bFjt0M+BRwVuy4D4APImxraJU6ArV/ayu/Gz+enbfailfXreOHDzzAJ377W5b+139taLNIvVPdEpHcHn/lFUZtu23WYwoZADtm5Ej2T8rua910UxobGmg04+W1a/uc++q6dVlv4JJdfPfdTFu6lPvOPLMkg2L5BP3if1/K+XenErIYRaQ6DHD218XAuQQzv54AvuDu75Wq7WFE1b9tNXTogPqyTP3YQK9ba9SfiUQvygBhK/Bi0utVJLID0zkHuCv2fCfgNeAGM9sTWAhc6O7rUk8ys/OA8wDaYpl8pVBpI1DjYlOH4w4YNoydrrqK3y9ZwtdSshFF6kk8KNixphMj+NQKqlsiksnzb7/NyC23zHpMIQNgmw4ZwqZDhvQ7bp/tt+fe5cs5Iel69y5fzud22SVnWy+86y6mLV3K/Weeyc5bbZXz+GLIN+gXXzAk/rfo4umLuWLOspIFCishi1FEKt9AZn+ZWSvwVWBXd+80sxnAycDvSvpN5BBV/za4sbHgvixbPzaQ69Yi9Wci0YsyQJguntV/KT/AzA4hCBAeGNs0CNgb+Iq7P2pmVwGTgO/2u6D7VILOifb29rTXj0KljkDFbTJ4MLttsw3/fuON0OeI1JrUYsapfyBUt0Skv153XnznHV58+21aN9uMBuvfnQ9kACzV1w44gDNuvZX9WlsZs8MOXLtgAavffZfz29s3HPPrxx7j1489xtMXXLBh25fvuIM/Pv44s04+mS2amzf0m5sMHswmgwez9oMPePbNNzd8TyvffpvFL7/Mh5qbsy74FUbqKsG5lLOweiVkMYpIVSh49lfMIKDZzLoISkOtjrzFeYqyfyukL8vVj4W9br1QfyYSvSgDhKuAHZJeDyNNR2FmowgWIxnn7m8knbvK3R+Nvb6ZIEBYMSpxBCrZe93dPP366xyy446hzxGpNemKGadS3RKRvi7Yd1/Onj2bnX75S9ZecglDBkW6nhkn7b47b3R28sMHHuCltWvZfZttuPO00/hwS8uGY15fv55lKQNe1yxYAMBhf/hDn+2XHnQQlx18MAtWr+aQ3/8+sf3++7n0/vs5c889+d348RF+R/2Vu7B6vgFNEalLBc/+cvcOM/spsBLoBO5x93vSnVSu2V8Qbf9WSF+Wqx8Le916ov5MJFpRfuqfD4wws+FAB0Ga+anJB5hZGzATOMPdn4lvd/eXzexFMxvp7suAw0gavaoElTYC9Y177uHoj32Mts0359V16/jBAw+w7oMPOHPPPTecE2U2hUglChP8U90Skb4O22knXrjoopK+53/tuy//te++GfdfdvDBG26W4vzSS7Ne8+Add8x5TDFlq3GqwuoiUgUKnv1lZlsQZBsOB9YAN5nZ6e7+p34XLNPsL4i+f8u3LwvbR+W6rohIsUQWIHT3bjO7AJhDUOj2endfambnx/ZfC3wP2BK4xoIAW7e7x6NgXwFujK1gvBz4QlRtLUSljUCteucdTrnlFl5fv56tN96YA4YNY9655/Y5p5KyKURKIVMx4zjVLRGRYsg1hViF1UWkCgxk9tengRXu/lrsmJnAJ4B+AUIREalckc4bcvc7gTtTtl2b9PxcgtWu0p27GKjY4gqVNgI17fjjc16z1NkUIuWWrphxfKGSVtUtEZEiyTWFWIXVRaQKFDz7i2Bq8QFmNpRgivFhwIKStFpERIom2sJCIiJlpGLGIhK1WYs6MmYqx6cQR/m3KNvU5oEcKyL1ZSCzv2KLSt4M/BPoBhYRm0YsIiLVQwFCEalpKmYsIlGJTy3OJHkKcRR/i/JZHbmcKymLSHUY4OyvSwFNVRIRqWIN5W6AiIiISDXKtlJ6KaYQZ5vaPJBjRURERKT+KINQREREpADZViGePGGPvDLzCpn+m8/qyFpJWURERKqJSqOUnjIIRUREisTMjjCzZWb2rJlNynDMwWa22MyWmtnfS91GKZ5MqxC3tjTnHRy8ZOYTdKzpxElM/521qKOg90+3PZ9jRURERMqp0M9GMjAKEIqIiBSBmTUCVwPjgF2BU8xs15RjWoBrgGPcfTfghJI3VIpm4tiRNDc19tmWa2rxrEUdjJkyl+GT7mDMlLkbRscLmf6bz/sX0lYREZFKk64fldqj0ijloSnGIiIixbEf8Ky7Lwcws2nAscCTScecCsx095UA7v5qyVspRZPv6sSZFgrJVMcw1/TffN5//OhWFrzwJn959EV63Gk043P7aBEnERGpHpn60QUvvMl9T7+mqag1RKVRykMBQhERkeJoBV5Mer0K2D/lmI8BTWZ2P7ApcJW7/6E0zZMo5LM6cabR8EYzetz7HR9m+m/Y95+1qINbFnZseJ8ed25Z2EH7hz+kmygREakKmfrRP81bueF1x5pOJt68BED9WxXbvqWZjjTBQJVGiZamGIuIiBSHpdmWGvUZBOwDHAWMBb5rZh/rdyGz88xsgZkteO2114rfUimLTKPePe6RT//VVB0REal2YbPHunqcy29bGnFrJEoqjVIeyiAUESkirbZV11YBOyS9HgasTnPM6+6+DlhnZg8AewLPJB/k7lOBqQDt7e39U8tCaGlr47Lnny/kVEnR0tZWlOtkGg1vjf2tiPJvh6bqiEitUP+Wv2L1Y+WWqR9N5631XYyZMlefxatUvmVcpDgUIBQRKZJMdVFAUxzqxHxghJkNBzqAkwlqDib7P+DXZjYIGEwwBfkXUTTmou9/P4rLygBMHDuyX83B+Gh4PlOVC6GpOiJSK9S/1a90/Wg2+ixenVITLn5x0l76+ZWIAoRoFKoQtTIKJbWjEjL3sk3hU6dW+9y928wuAOYAjcD17r7UzM6P7b/W3Z8ys7uBx4Fe4Dp3/1f5Wi2lVM7R8GzBSRERkWqQrh99c937dHb1ZjxHn8WrixIuyksBQjQKJVKNkgOCLUObWPteN129wUzMcnUkmsIn7n4ncGfKtmtTXl8BXFHKdknliDpTMNv7gqbqiIhIdUvtR2ct6mDiTUs23Aeko8/i1UMJF+WlAKGIVJ3UkaW31nf1O6YcHYmm8IlIJStXcFJERCQqyQNgmeoTFvJZvBJmJ9UjJVyUl1YxFpGqk25kKZ1SdyRabUtEREREpLTGj27l4UmHcuVJe/X7LG4Es4vGTJnLrEUdoa4XT0boWNOJk5idFPZ8KVymYK4SLkpDAUIRqTphA3+l7kjGj25l8oQ9aG1pxghWJp08YQ+NNopIWcxa1MGYKXMZPumOvG6MREREqlHyZ3EIgoPxicf5BPmyTXOVaCnhorw0xVhEqk7L0Ka004qTlasj0RQ+ESm3WYs6uPy2pX3+TqrIt4iI1JJMU4DjjzFT5vabchy2BJGmuZaPaiaXlwKEIlJ1PEMNYot9VUciIqVQifWJUmu0Jku9MYq3v2NNJ41m9LjTWiHfh4iISCZhVrodSJBPdcXLSwkX5aMAoYhUnbc7M2cPrphyVAlbIiL1KszNSTnkqtEav+FJbX+Pl3cVeBERkVySB7ZSpQ6CDSTIN3HsyH6DbZrmKvVANQhFpOqoeK2IlFul1ifKlRnRaEGudbZAYiV8HyIiIsmSFw7JJHlf2Fp26er1qq641CtlEIpIVZm1qIP1H3T3265RPREppUqtT5QpYyIunimYq53l/j5ERKT+ZCvdkStDHoJyQ/EAX7padofsvDVXzFnGxdMXb3h9y8KOjLMBFBCUeqMMQhGpCrMWdTD6+/dw0fTF/RYoaWlu0qieiJRUpWYyp8uYSBZf2TFXO8v9fYiISH1JzhB0+q86HGbgyqFPBvz40a08POlQVkw5ioljR3LLwo4+179x3sqKnA0gUi4KEIpIxYt/YMi0cvHGQwYpOCgiJRV26lKpxadFbTG0qd++5PZlCyRWwvchIiL1JVfpjrADV8mBxOTpw1+fsaTf9TOse6gseqlbmmIsIhUv15QCdeIiUmrppi6Vc/Xf1GlZlx69W9b2JbdfqxiLiEi55SrdkW7hkHTigcRMi3GFoSx6qVcKEEpdy1bnQipHrgCgOnERKYdKqU+UaUXlyRP24OFJh2Y8r1LaLyIikqmGboMZwyfdwfYtzXxun1b+NG9lxmskZ8CHqVkIQd3C5NChsuilnmmKsdStXHUupHJkCwCWohNPt7qZiMhAFetvS6WuqCwiIhJWptIXPe59agY2N6UPYTSa9alJHmaGUXNTI6cd0KbVikVilEEodSvbDZU6hcqSaUpBS3MTlx2zW6Q/r0yZOYB+T0SkYMX821KpKyqLSHUxsyOAq4BG4Dp3n5Ky/zTgW7GXa4EvufuS2L4W4Dpgd4KErLPd/R+lartUv3jfd9nspazpTF933IHOrl6aGoyu3kTeX3NTY7/AXqaMxEYzet01e0wkDQUIpW7phqp6lLPWlwLJIhKFYv5tyXQTpPILIhKWmTUCVwOHA6uA+WY2292fTDpsBXCQu79lZuOAqcD+sX1XAXe7+/FmNhgYWsLmSw15v7s35zGbbDSIoYMHZb0vSJdgkC6QKCIJChBK3dINVXUpV62sYgaSVfNSROKK+bflkJ23TluT6ZCdt877WiJSt/YDnnX35QBmNg04FtgQIHT3R5KOnwcMix27GfAp4KzYcR8AH5Sk1VJTwtYNXLO+i0Xf+0zWYyptMTGRahBpgHAgaeqx/Y3AAqDD3T8bZVul/mQaVVJRWklWrECypiqLSLJiDlLd9/RreW0XEUmjFXgx6fUqEtmB6ZwD3BV7vhPwGnCDme0JLAQudPd1qSeZ2XnAeQBtbW1FaLbUkrCDZGH7Si3GJZKfyBYpSUpTHwfsCpxiZrumHBZPUx8F/IAgTT3ZhcBTUbVR6tv40a1MnrCHitJKVukKJhcSSNYiAiKSrFh/W0AlM0SkKCzNNk+zDTM7hCBAGE/0GATsDfyvu48G1gGT0p3r7lPdvd3d27feWlnO0leYwJ8SOkSiE2UGYcFp6rHjhwFHAT8CvhZhO6WOaVRJcinW9ATdwItIsmJOfVLJDBEpglXADkmvhwGrUw8ys1EEi5GMc/c3ks5d5e6Pxl7fTIYAoUg26WZ4NTUaGw8exNudXZomLBKxKAOEA0lTB7gS+CawabY3UZq6iEStGIFk3cCLSKpiDVKpZIaIJDOzCcCPgW0IMgMNcHffLMtp84ERZjYc6ABOBk5NuW4bMBM4w92fiW9395fN7EUzG+nuy4DDSEoKEQlLdQNrl2qxV4coA4SFpKkfGHv9WeBVd19oZgdnexN3n0psanJ7e3va64uIlJtu4EUkKrqhEpEUPwGOdvfQpZrcvdvMLgDmENSPv97dl5rZ+bH91wLfA7YErjEzgG53b49d4ivAjbEVjJcDXyjadyN1RTO8ao9qsVePKAOEA0lTHwMcY2ZHAhsBm5nZn9z99AjbKyISGd3Ai0iUdEMlIkleySc4GOfudwJ3pmy7Nun5ucC5Gc5dDLSn2yci9S1bLXZ9dqksUQYIB5KmfglwSeyYg4FvKDgoUVPas0Rt/C5NjH/tu9CzHj7zD2jcqNxNEhERkdqzwMymA7OA9+Mb3X1m+ZokIvWq0muxKw6QEFmAsAhp6iIlo7RnidSKG+EfKWMcvR8oQCgiIiJR2AxYD3wmaZsTJGaIiJRUJddiVxygrygzCAeUpp50zP3A/RE0T2pcmJGA+DHp/mAp7VkG5P034aET4JW5fbfvNQV2+SZYujKtIlLLNEItIqXg7qr/JyIVY+LYkUy8eQldPYklI5oarSJqsWv6c1+RBghFyiXMSEDqMekUK+1ZN4V15JHT4fkb+24bOgwO/Rts9rHytElEyk4j1CJSKmY2DPgVQV13Bx4CLnT3VWVtmIjUr9TlZCtkedlKn/5cag3lboBIFLKNBGQ7JlVq2vOsRR2MmTKX4ZPuYMyUucxa1JGzLfGbwo41nTiJm8Iw50qVWPMv+LMFj+Tg4B6XwSk9MP5FBQdF6lyYfklEpEhuAGYD2wOtwG2xbSIiJXfFnGV09faNCHb1Ol+fsSSv++ooZJrmXAnTn8tBAUKpSWFGAnKNCjQ3NfZJey400KebwhoWDwreuUff7QfOgFMd9rgUTH9mRUQj1CJSUlu7+w3u3h17/A7YutyNEpH6lOmzTo972RNoJo4dSXNTY59tqXGAeqI7V6lJYUYCso0KtLY0M3nCHn2mfWUK9OUa+dBNYY157R+JwGCqk94PAoNtJ5S+XSJS0aIYoS4kq11E6sLrZna6mTXGHqcDb5S7USJSn8J81ilXAs340a1MnrAHrS3NGOnjAPVEAUKpSWFGAiaOHUlTY/8gT1ODpa0RWOjIh9KWa0Q8KHjvJ/pu3+bgICh4qkPj4LI0TSqHmR1hZsvM7Fkzm5Rm/8Fm9raZLY49vleOdkrpFXuEWuUrRCSLs4ETgZeBl4DjY9tEREou3WegdMqVQDN+dCsPTzqUFVOO4uFJh9ZtcBC0SInUqPh/6mwLg4wf3cpls5eyprOrz7ldvZ521aJMy7Mn6+zq4bLZS/u87yE7b80tCzv6ZB/Wc9pyVVnzr/7Th+MmvAobabaOJJhZI3A1cDiwCphvZrPd/cmUQx9098+WvIFSVmH6pXwUe9W91MW0Dtl5a+57+jUtriVShdx9JXBMudshIgL9PwM1mNHj/VcpUQJN+YUKEMZuerZNPj7W8YhUrPGjW9PezCTfBGVaPCnd6MXEsSNzrnoMsKaza0PQsWNNJ7cs7OBz+7TqRquapJs+HHdqhSy5JZEqsN/bD3jW3ZfHrjENOBZIDRDWr+T/WyO+DPv+unxtKYNM/VIhilm+It0Ky3+al/h171jTycSblnD5bUtZs75L/ZhIhTKzb7r7T8zsV6RZI9Tdv1qGZomI9PkMlPq5A5RAUylyBgjN7CvApcArQG9sswOjImyXSCTS/TFKJ93oRdiRj1SdXT3c9/RrPDzp0MIaLaWxfjXMynCze+wLsHFbadsjZTOAfq8VeDHp9Spg/zTHfdzMlgCrgW+4+9I0bTgPOA+gra2GfvcOvAkeitXo/PfVwQPgmOWwyfDytasKZcpqL2T0PV02YqquXuet9YnBr0tmPtHnfA2AiVSEp2JfF5S1FSJS9+JJOR1rOmmM3Te3pnxO0OeHyhMmg/BCYKS7q7CtVL0wN0HNTY0csvPWjJkyl9VrOmkZ2oQ7vN3ZN2sibLARtCBJRZv9EVi7PP0+ZQvWq0L7vXSpp6m/RP8EPuzua83sSGAWMKLfSe5TgakA7e3ttfOL2HZ88P/qud/Co+cmts/eKfFc/+9CSZfVXujoeyF9VGdXD5fftpT3unr7ZB7GA4f6kC9Seu5+W+zpene/KXmfmWkFNREpidT75HhSTernBH1WqDxhAoQvAm9H3RCRUsh2E2SQtmZgPGMC0t/8JI98rP+gu8/xcaqnUGHefxNu2TL9PmUySeH93ipgh6TXwwiyBDdw93eSnt9pZteY2Vbu/npBLa1WHzknePR2wbSUxX3i05APuh1ajyp926pEMUffw9TYTSddfzeQOogiUjSXADeF2CYyYKk1bJUJJtmScvQ5obKFCRAuB+43szuA9+Mb3f3nkbVKJCKZboJaW5o3TAEeM2Vu1qzA5D9qqSMfqqdQ4aZvDD3r0+9T1pIkFNrvzQdGmNlwoAM4GTg1+QAz2w54xd3dzPYDGoD6zdBvaEr833v6SvjnxYl9f09ax+WUXrAstUHrVLFG38PW2A1LWfMi5VROz2EAACAASURBVGFm44AjgVYz+2XSrs2A7vK0SmrRrEUdXH7b0n4DRR1rOrlo+mJuWrCSG//z40V/TwUiq0OuzwFRf07Q70rhwgQIV8Yeg2MPkaoVZkpWmD9YmY5RPYUKlC1b8Mh/QctupW2PVIOC+j137zazC4A5QCNwvbsvNbPzY/uvBY4HvmRm3UAncLJ7iGKm9WDni4KH98JfGvvu+0tD8HXf/4UR55e+bTUuXd+VvIrx5s1NrPugm66exK9qc1MjQwY1bFiUK5my5kXKZjVB/cFjgIVJ298FLk57hkieZi3qYOLNS/r0Cakefu5NvjPrCX44fo+ivWfqYloqaVG5cs1MiPJzQrF/V+ot2Ghh70vMbFPA3X1ttE0qXHt7uy9YoJq8kl2u/+RjpszNOdUqOeNQKpRWIi4ZM1vo7u3lbkexVUq/V9d924u3woMT0u876X1o1LhlqaTrO4G0g26TJ+xR0x+epX5Ua/9mZk3u3j96X2Hqun+rYmHulQAaDJZPLrxUSHK/k2lxSN2TVaZstfqj/pyQ6fezkN+VTLMDa+FzTqb+LcwqxrsDfwQ+FHv9OvD5dKsuilSDXFOyck210pThCta9DmZskn7fJ2+BHTIEGkSSqN+rIDsclwjopwb9pw8Jvu78ddj7p6VtVx3K1nfW08i6SJXY0cwmA7sCG8U3uvtOmU8RCSfs9NDeAYzHZ1rkotC2SGklz0zItopxFDL9ThTyuxKvpdhkXXR5E1D7NRTDTDGeCnzN3e8DMLODgd8An4iwXSJlM350KwteeJO/PPoiPe6YQfOgBjq7enXzU6lubYXO1en3KVtQ8qd+rxLF/y+//ijcc0Bi+9M/Cx4Ax78Fg1tK37Y6plUIRSrSDcClwC+AQ4AvEKzHJ5JVmOmUmzc3pS0vUUzZFrlIVo0lLQ7/+f38+9V1G16P2GZj7v3aweVrUETK9fkg0/TmUL8r7z4HL84MHm/M4+E2oC3Y9dWV32D2moOB2g5MhwkQbhy/SQJw9/vNbOMI2yRSVrMWdXDLwo4NI1Xu4Bi/OGkv3QRVknSrn8btNxU++p+lbY/UEvV7lWyr/RPBwlu2gfdfS+y7eYvg6/ZHwcG3l75tIiKVodnd/2Zm5u4vAJeZ2YMEQUORtMLWbgu7ZlhzU0PBbQkTgKnGWV2pwUGAf7+6jsN/fn9NBgnLIeeaA71dcPsusPa50Nd8o3szHnh37w2vqzEwHVaoVYzN7LsE060ATgdWRNckkeLKt7BouhGrWk8lLkTZCrbO/zL8+5r0+5QtKMWhfq9afO7V4Ova5TD7I4ntq+9ITEk+6knYfJfSt01EpHzeM7MG4N+xxbM6gG3K3CapcGHvgdasz5092ABMnjCq4LbkWuSiFFNVo5AaHMy1XfIX/53489/uY8b2ZyR2PBV7ZDNoUxh+RlCWaptPMWvJqzkXOK01YQKEZwOXAzMJUtMfIEhTF6l46UbCLp6+mIumL87YsRSzbkGtKvlKYu6JVUxTjfoh7P7fxX9PqWfq96rNJjslBgj+9ml45W+JfXfsmnge8SBCpa50V6ntEpHIXAQMBb4K/AA4FDizrC2Sihf2HihX8M6AUw9oG1A/M3HsSC6evph0vbYWJpE+Vt8N94/rs2k8MH77EOce8PsgIJghLTa5lmK9fIbKGSB097cIOheRqpNuJCze0WQKag2obkGdKFmW5Qsz4OGT0u9TtqBERP1elTvsr8HXzpfh1v/ouy+eVXjQbdD62aK+bckHTqq8XSISHXefH3u6Fg1wSUhh74FyLejowH1Pv5Z2X1jjR7dy0fTFafcpaaNOLZ4ET/64sHN3/hrs/bOCTq23WssZA4RmdqW7X2Rmt0H/4L27HxNpy0SKIFcHki6olbNugUSbZZktW/CgO6D1yIG/h0ga6vdqTPN2iYGEu0bDW0k3Gn8/OvG8SIMNlVqeolLbJSLRMbP7SN+PKe1KMgp7D5S6Qm068XuCgWSwtyppo/70vA/TN8p9XCaHzIH/+Ezx2lOHsmUQxmsv/bQUDRGJQq4UeOgf1KrHVOJ8RZJlufouuD9D8O+U3vAVkUUKp36vVo1bFHztXgczNum7L55VOGYafDhDxnIIlVSeIvmGLFP4UxkYIjXtG0nPNwI+B3TnOsnMjgCuAhqB69x9Ssr+04BvxV6uBb7k7kuS9jcCC4AOdy9umrZELp97oHhW1Zgpc9PeE7QMbRpwBnstJm1sNqSRd97vn3m52ZDGMrSmjN5+qm8JmHx97nUYsmXx2iMbZAwQuvvC2NO93P2q5H1mdiHw9ygbJlIMuVLgIX1Qq95SifNV1A77zxkCf2Omw4dPLLCFIvlTv1cHBm2cyBh8/DL41+WJfQ+fHDwg9KBEciCuwYwe7x+OK2Wmw6xFHVx+21LeClFAXhkYIrUrqT+Le9jMsvZhseDe1cDhwCpgvpnNdvcnkw5bARzk7m+Z2ThgKrB/0v4LCZYB2Gyg34OUR773QBPHjmTizUvo6unb/619r5vLb1s6oAz2WkzaePzyIxh16d19goSbDWnk8cuPKGOrIvTqQ/DXTxZ+/ik9YIWvhi35C7NIyZkEI0nJzkqzTaTipKbAG33nW1T7KFS5DLjDfvVB+Oun0u9TtqCUn/q9ejDqsuDhvfCXlJH7eJmDnb4AB1yf9vTUzIh0wcGo+ph0U7aAnANiUbdLRCqDmX0o6WUDsA+wXY7T9gOedfflsWtMA44FNgQI3f2RpOPnAcOS3nMYcBTwI+BrA2m/VI/xo1u5bPZS1nT2HZjq6vWMg1X5ZLDXYtJGTQYDn70OHvvPws7dYQJ88pbitkcKlq0G4SnAqcBwM5udtGtT4I2oGyZSLMkdy0DqYGgVyL4K6rAzZQvu9xv46LkDb5TIAKjfq1PWkMgqXPFH+MfnE/uW3xA8AE56DxqHbNiVrrYfQKMZve6R9ROZpmwNGdSQMzhooP5LpD4sJBgTN4KpxSuAc3Kc0wq8mPR6FX2zA1OdA9yV9PpK4JsEfWZGZnYecB5AW1tbjiZJOYW993m7M3fWejJlsFep3h549BxY8fvCzv/UbBh2dO7jpKyyZRA+ArwEbAUkL/nyLvB4lI0SiUq6oFaYzk+rQA7Amwvh7vb0+07uhoY6q7khlUz9Xr0bfkbwgP4DGvGi2TueBp/4U8YMiF53Vkw5KrImZlp0JFdwsLWlmYcnaX0CkXrg7sMLOC3dKG7aMqZmdghBgPDA2OvPAq+6+0IzOzhH26YSTE2mvb29OKtESdHlc++TqTZ5S3MT73f31lQNwWoXKuj73uswZ19Y93z+b7DFaDjsPhi8eVHaK6WXrQbhC8ALwMdL1xyR0grb+WkVyALctDl0vdN/++ifwi5fL317RHJQvyd9xLMKX/k7/O3gxPbnb4Tnb2TFKNj3yT/yWvcWfU6LOjOikMVFdEMmUh/MbEK2/e4+M8vuVcAOSa+HAavTvMco4DpgnLvHs+vHAMeY2ZEEi6JsZmZ/cvfT82m/VI5M9z5fnxGsSZN8/5OpNvllx+y24VqagVV+qfe9W76/hPFPHRZUDc3XyAth75+rPmANylmD0MwOAH4F7AIMJljVap27q/isVL2wgb9KWp2yomVbkeqk96Fx8IAur2neUgrq96SPbQ9KBAtTsgrn73rGhuc7Pn57SQJxmTI1thjaxHtdvf36tJbmJi47Zjf9rRSpD9nm7zmQLUA4HxhhZsOBDuBkgrIbG5hZW+waZ7j7Mxsu7H4JcEnsmIOBbyg4WN0y3eP0uPdLpshVm1z9T5m9/Df4xxmM73yJ8bvkee6YafDhkyJpllSmMIuU/Jqgg7gJaAc+D3w0ykaJlErYwF+mGzLV0IiZPQLWPtt/+27/DXv+sChvUaxp3goySgjq9yS9eKDwrcfhrj377Hp+1GeDJ1vcSMo9dVFlytS49GhlaojUO3f/wgDO7TazC4A5BANj17v7UjM7P7b/WuB7wJbANRYsKNft7hnqyEg1y3TvA+mTKWpxMZGq4r3w72thwZcLO//IJ6Bl9+K2SapSmAAh7v6smTW6ew9wg5k9kvMkkSoQNvCX6YasrqdsvfcazNwm/b4T18GgoUV9u2JM81YtSQlL/Z5ktcWojFmFPHJa8IDEMUWkTA0RCcPMjgJ2I5jyC4C7fz/bOe5+J3BnyrZrk56fC2RdVc7d7wfuz7vBUlHS3fsk0yyqMul6FxZfAv++Ou9Tb3znRH7wwom85xv12d7a0szDCg5KTJgA4XozGwwsNrOfEBRw3zjMxc3sCOAqglGo69x9Ssr+04BvxV6uBb7k7kvMbAfgD8B2QC8w1d2vCvOeIvkIG/jLdUNWV5Z8B5b+qP/2EV+GfX89oEtny+4rxjRv1ZKUkAru96QOxYOAb/4T7t6n77548HDTEXD0MxRLPWRq7Djpjn7bno9w8ReRWmJm1wJDgUMI6gUeDzxW1kZJVYn3MV+fsYQe7z/YpVlUEXv3OXjsP+GV+/I/94AbYPiZYH0HMDde1IF1PAFKeJEswgQIzyAI8F0AXExQvPZzuU4ys0bgauBwgqK3881strs/mXTYCuAgd3/LzMYRrGi1P9ANfN3d/2lmmwILzezelHNFBiyfwF893JBl9MEauHmL9PuKlC2YK7uvGNO8VUtSQiqo35M696G9M2cVvvvvxLYIsgprTbrgYHy7goQioXzC3UeZ2ePufrmZ/Yzs9QdF+ogP2ve4Y/RdzlpBpSJ6eS7843TofCm/8zb5KHz8d7D1mNCnKOFFwsgZIIyt6gjQCVyex7X3A5519+UAZjYNOBbYEORz9+QpW/MIVsvC3V8iyNjA3d81s6eA1uRzRYqlrgN/uTz5E1j8rf7b974Sdr6wqG+VK7uvGNO8VUtSwhhAvycSiAcB16+CWTv03RcPFG5zEHz6/pI2S0TqRvzDznoz2x54AxhexvZIFUkdtHfYECRsVVApfwOpD9h6NLRfDRvvkPvYEHTfK7mEWcV4BX0HDQBw951ynNoKvJj0ehVBdmAm5wB3pXn/HYHRwKM53k9EiqFrLdy0afp9J7wNTdEs5Joru68Yo16qJSlhDKDfE+lr6LDMWYWv/j2x7ZQesIbStk1EatntZtYCXAH8k6BP+015myTVIt2gfTw4+PCkQ8vTqGrw3uvw10/CO0/nf+4u34Q9Li16DXeRfIWZYpy8MtVGwAnAh0KcZ2m2pZ1XY2aHEAQID0zZvglwC3CRu7+T4dzzgPMA2traQjRLRNJ65pr0I1ujfgC7fyfytw+T3TfQUS+l1ktIhfZ7IpnFA4W93TCtqe++vzQGX5tb4bhVpW2XiNQcd/9B7OktZnY7sJG7v13ONkn1UEmeHN5YAHP2Lezc/a+Hnc7qVx9QpFKEmWL8RsqmK83sIYJl7rNZRVC3KW4YsDr1IDMbRVA8d1zye5lZE0Fw8EZ3z1gzw92nEtQupL29XYV9RPLR8x5MzzC99nNvwJDSxURKld2n1HrJZQD9nkhuDYMSwcJ7xsDrSdVWOjsSWYUnroVBWhtHRPJnZkuA6cB0d38OeL/MTZIqopI8MZkWZsylcSM49K951QeU7LItZCnFFWaK8d5JLxsIMisyzD/sYz4wwsyGAx3AycCpKdduIyiYe4a7P5O03YDfAk+5+89DvJeI5GP572HeWf237zIRRv+k5M2B0mX3qYORXAbQ74nk5zMPJ56nTkGesUnieZ0tbPL8lKO0irHIwBwDnATMMLNegmDhDHdfWd5mSTWoq5I8vV0wbXDh5x/5BLTsXrz2SD+5FrKU4gozxfhnSc+7geeBE3Od5O7dZnYBMIdgNcjr3X2pmZ0f238tQTbGlsA1QUyQbndvB8YQrCL5hJktjl3y2+5+Z6jvSqQAlRY4Knp7snWAx70EzdsVfu0iiTq7Tx2MhFRQvycyIPEg4HPXw6Pn9N0XDx4eu7JohcornYKBIoWLLbb1E+AnZjYC+C7wY4J7MpGsarIkzzvPwO0FBjg33w0OfwgGtxS3TRJKroUspbjCTDE+pNCLxwJ6d6Zsuzbp+bnAuWnOe4j0NQxFIlFpgaOitmflzfDQCf23f/SLsN+1/bfXsCg6mEoLLMvADaTfExmwj5wdPKB/VuH/JdVarrOsQhHJT2yhxxMJMgl7gG+Wsz1SXaq2JM+TP4bFkwo790P7wth5WjSsiIpxn6SamKWVMUBoZl/LdqKm/kotqbSRiQG3p7cnVgQ/zQ3ksS/AxvWzoE9yx5TpdrrQDqbSAssyMOr3pOLEg4CvPhSsjJgsHjw89F7Y7tOlbZeIVDQzexRoAmYAJ7j78jI3SaS4bm2Fzn7LG4Sz/ZFwcP8yFlJcxbpPUk3M0sqWQah6S1I3Km1kImx7vjPrCf786Ep6Y/eQh7f8k9+0pVlH4cOnwJg/F7uZFS+1Y8qk0A6m0gLLMmDq96QybXNgIliYmlU49/DEc2UVikjgTHd/utyNEBmQgdYH3PcaGPGl4rVH8lKs+6S6qolZATIGCN398lI2RKScKm1kIkx7vjPrCf40byVGL8+POib9hT77DGw2IqpmVrx0HVOqsB1MuhT5Sgssy8Co35OqEA8CvvNvuP1jfffFg4d7TobdCpxiJSJVy8yudPeL3P1pM7vQ3a9K2vc7dz+rjM0TSW/tcpj9kcLPH7cEthhVvPZIURTrPqkma2JWsDCrGG8EnAPsBmwU3+7uZ0fYLpGSqrSRiTDtefaJ23l+1Lf7nXvfO/tw8Sv/w+JLP1OStlaybB2QQegOJlOKfMvQJt5a39XveKW8V7eB9HtmdgRwFUEh+OvcfUqG4/YF5gEnufvNxWi31JnNRiSChTM2he61iX1LLgkeoKxCkfryqaTnZxL0R3GKoEjJpK0913I//OP0wi96cjc0aJ2dalHMBJyqrYlZhcKsYvxH4GlgLPB94DTgqSgbJVJqlTYykbE9e20fjLCtXc60lIG2I575FU+/Nzz2qn/Qql4kfyBpMKPH+98ct7Y08/CkQ0NfM1OK/JBBDTQ3NVZMYFmKpqB+z8wagauBw4FVwHwzm+3uT6Y57sfAnCK3W+rVie8GX7vehZs267svnlW40XYw4aXStktESs0yPBcpma5pmzG+913GtwHxsudhowd1WhapFlVaAo6EEyZA+FF3P8HMjnX335vZn9FNjdSgShuZ6NOe1x+Fe4b161z/uW4kE577WekbV6FSM/3SBQcL6ZgyZSK+3dnFL07aq2ICy1I0hfZ7+wHPxovBm9k04FjgyZTjvgLcAuxbzEaL0LRpImPw1mHQ2ZHY997LiWDhKT1apVGkNjWY2RZAQ9LzeKBQqVdSPN3rYcbGaXc1hTn/wJuh7XNFbZJUlkpLwJFwwgQI46lIa8xsd+BlYMfIWiQiCTdtAV1r+m8fO58dr3gl42lbDA3VNdecTDUHG83odS+4Y8qWIl9pgWUpikL7vVbgxaTXq4D9kw8ws1bgOOBQFCCUKB23Kvja2wPTUj7u/SUWJ2hogpM/KG27RCRKmwMLSQQF/5m0T/UGJH9vLIA5hX9c+eTT1/HiB9sBwS/liilHFalhUg2KcZ+Udrr6AK5Z7OvVmjABwqmx0afvArOBTWLPRSQKL90L96WpH7jZLnDUUrDgM99mQ+7mnffTL8Bx6dG7RdnCipUp06/XfUAfSJQiX3cK7ffSTedKvSG7EviWu/eYZZ79ZWbnAecBtLW1ZTxOJKeGxkRW4Z17wprHE/t6uxJZhcevgcGbl759IlI07r5judsgVeq1f8C9nyj8/JPeh8ZgxeExU+ZW1OKPUr0y1YEHCgrqFft6tShMgPAGd+8B/g7sFHF7ROrXnzMECw6bC9se0m/z45cfwahL+wcJrzxpr7r9AxfVatRKka87hfZ7q4Adkl4PA1anHNMOTIsFB7cCjjSzbneflXyQu08FpgK0t7cr66OKVPTI9JFLEs9T+5ybWxLPtbCJiEhtWv57mHdWwafv+PjtQP963sl93+bNTTQ1Gl09ib5EA+u1KerPPJnqwF8xZ1lB71Ps69WiMAHCFWZ2NzAdmOuepqiXiBTmtUfg3jHp94WoEfX45UdE0KjqFWWmXzxFPt4RXjx9MVfMWVZZN/9SLIX2e/OBEWY2HOgATgZOTT7A3eMrCWFmvwNuTw0OSvWqqpHpeBDwXz+Cx7/Td188eHjMcthkOCIiUkV6e+DvR8NLdxV2/iH3wH8cDsDwSXeknZuePGsnte9b09lFU4OxxdAm1qzvqrzBMimKqD7zJAcdM30AzzRrLJdM5xV6vVoUJkA4Ejga+DJwvZndBkxz94cibZlILcuULbjvNTDiS6VtSw2JOtMvXUc48eYlXDZ7KW936gNQDSmo33P3bjO7gGBBk0bgendfambnx/ZfG3G7pcyqcmR69/8OHtC/b5qdlECrrEIRkcrS+TLc+h+Fnz/hVdho66yHhJmdk67v6+p1hg4exKLvpSmbJDUhis88qfdamRQ6Oyyq2Wa1JGeA0N07gRnAjFhNpqsIpl1pJSyRfLz6APz1oPT7Tu6ChjDxesklykVD0n4A6nHWdAZrWlR0tpCENpB+z93vBO5M2ZY2MOjuZw24sVJRqn5kOh4EfOU++NuhfffFg4djH4Mttb6OSCUzsz+6+xm5tkmVeHMR3L13YecOOxY+ObPglevDzM6p+r6vSCq6xEgEovi5Z1pwMtlAZoeprnxuoSISZnYQcBIwjmAK1YlRNkqkpmTKFtxzMuw2qbRtkYzCdOphOryKzxaSUNTvlU4tfaCumZHpbQ9JBAtT+7A5+yWeK6tQpFL1Wa3OzBqBfcrUFglr+R9g3pmFnbvLN2D0FcVtD+Fm59RM3zcAVVVipEii+Llnu9ey2LUH8jlRdeVzyxkgNLMVwGKCbIqJ7r4u8laJVLs3F8Ld7en3nfQeNA4pbXskaxAiXad+0fTFXDZ7KZcds9uG4zJ1hKnqbcS01qjfK51a+0BdkyPT8SDg+lUwa4e+++LBw6R6VSJSPmZ2CfBtoNnM3olvBj4gtvCVlJn3wmNfhOeuK+z8cYtgi72K26Yccs3Oqcm+L09VWWJkgKL4uWe610pdFGcgopxtVgvCZBDu6e7v5D5MRDJmC25/JBx8R2nbIhvkCkJkSmdf09nV57h0HWE69TRiWqPU75VIrX2grumR6aHDEsHCvx4Mr/49se++pBpTyioUKRt3nwxMNrPJ7n5JudtT17rWwpJvwzO/yv/czXeFwx+CwVsUv10RqOm+L6RyTbMu5yyMKH7uCjaXX8YAoZnNcPcT3f0dM/uxu38rad897q6KoyKQvS7ICW9D02albY/0kysIka3zTj4utSNsGdrE2ve66epN3BCrE6te6vdKrxbrFtXFyPSn7w++dnfCjKF998UHyj71fzDsmJI2S0Q2uMvMPpW60d0fyHaSmR1BUHe3EbjO3aek7D8NiPeNa4EvufsSM9sB+AOwHdALTHX3qwb+bVSR534Lj56b3zkf/WKwQGGB9QGjUkjQqS76vizKMc26EmZhFPvnrmBz+WXLIByR9PxwEp0BQPbljkTqQaZsQQxO7S1pUyS7XEGIXFOHk89P7QhrqX6aqN8rNdUtqnKDmhMZg8t+BQu/mtj3wLGJ56f0gmXqM0UkAhOTnm8E7AcsBDLO0YvVKbyaoP9bBcw3s9nu/mTSYSuAg9z9LTMbRzBteX+gG/i6u//TzDYFFprZvSnn1rZswcEDfg87fb50bRmASgg6VaNyZL7V2iyMuHoPNpdbtgBhtjkimj8i9SlbbcHxHTB0+9K2BwWowsgVhMg1dThbsEKdWE1Rv1dimkpSQ0Z+JXi4w19SsmHir/f5ZXCMiETK3Y9Ofh3L8PtJjtP2A5519+Wxc6YBxwIbgnzu/kjS8fOAYbHtLwEvxZ6/a2ZPAa3J59a8498EDAa3lLslA1KrQaeolSPzrRZnYRST7pELky1AONTMRgMNBIVuRxMUuTVAQ/tSXzJmC1LWekvFHuWr1T+kuYIQ8e/x8tuW8tb6rj7nKlhRV9TvlZimktQgs0S/uGp230zChV9NZBlqwS6RUloF7J7jmFbgxZRz9s9y/DnAXakbzWxHYDTwaLqTzOw84DyAtra2HE2qIlVSLzAXBZ0KV+qkAc3CyEyZsIXLFiB8Cfh57PnLSc/jr0Vq2zv/hts/ln7fkU9AS67PWdEr5ihfLf8hDROEiHfqtRoklVDU75WBsnBr2LBjEsHC1IG26RsFX/e/Hj7yhdK2S6TGmdmvSGS+NwB7AUtynZZmW9pRcDM7hCBAeGDK9k2AW4CLMi325e5Tia2o3N7eruz8CqOgU/WopVkYxb7/UiZs4TIGCN39kFI2RKQUQv3xqdBswXSKOcpX639IwwYhFKyoX+r3RCIU7z/fmA9z9ktsf/Ts4AFwwrvQtEnp2yZSexYkPe8G/uLuD+c4ZxWwQ9LrYcDq1IPMbBRwHTDO3d9I2t5EEBy80d1nFtpwKa9aCjrVulqZhRFFkooyYQuXLYNQpKZk/ePzsV74vwzTHA79K2x3WNrrlfsPcjFH+fSHVEREIrflvolg4WNfhGenJvbdtGnwddQPYff/Ln3bRGqEu//ezAYD8akwy0KcNh8YYWbDgQ7gZODU5APMrA2YCZzh7s8kbTfgt8BT7p6cfS9VplaCTvWiFhIbokhSUSZs4RQglLqR7o/PU7uMg6cIHqmyZAtWynTcYo7y6Q+piIiU1H7/L3i89yrM3Dax/fHvBA+AY5bDJsPL074qUgmDllI5zOxg4PfA8wRTh3cwszPd/YFM57h7t5ldAMwBGoHr3X2pmZ0f238t8D1gS+CaICZIt7u3A2OAM4AnzGxx7JLfdvc7o/j+JFq1EHSS6pEpGaVjTSdjpswtqF/LtADl+g+6mbWoQ7/fWShAKHUj/sdn44b1LN39xPQH7f0L2PminNeqlOm4xRzl05QCEREpi422SQzK/etHieAgYrfhWQAAIABJREFUwOydgq8f+U/Yf2r/c6ViBi2lovwM+Iy7LwMws48BfwH2yXZSLKB3Z8q2a5Oenwucm+a8h0hfw1BEJKtMSSoGG7bn26/Fj7ls9lLWdCYWoHxrfZf6xxwKChCa2c7u/nSxGyMSpTtHfp1dhmSYYZFnbcFKmo5brFE+TSkQCZhZk7t3pWzbyt1fL1ebROrG7v8dPLrXwYykeoTP/SZ4ABz9b9j0o+VpXwWqlEFLqShN8eAggLs/E6sRKCJSUdIlqRj9V0jKt18bP7qVK+Ys6xMgLOQ69abQDMJ7gBpal15qVs/7G1ZK3GVI311fW3kxd637DJMn7MH4PC9bq9NxNaVA6llsVcY/AkPMbBFwnrs/H9t9D7B3udomUncGbZwYvFv+O5iXtNLxbSOCr9t9Gg69t+RNqzSVNGgpFWOBmf2WoE8DOA1YWMb2SMRUZqB66GfVV7oklXT32ZC+X8v27xnF9OValzFAaGa/zLQLaImmOSJFcu+n4LUH0+4as/JvG/4YTJ6g6bgissFPgLGxmkvHA/ea2RnuPg9NnRIpn53OCh69XTBtcGL7y3+FP8f+a459LFgApQ7V6qClDMiXgC8DXyXovx4ArilriyQyKjNQPfSzSi81SWXMlLmh+rXvzHqCG+et3JBtmPrvGcX05VqXLYPwC8DXgffT7DslmuaIDEBvD0zL8Cs96gewe1DT6OEivJWm44rUpMHuvhTA3W82s6eAmWY2if4zHUSk1BqaElmFq++C+49M7JuzX/B1aBsc+zxY/cT0NWgpqdz9feDnsYfUOJUZqB76WYUTpl+btaijT3AwLvnfM6rpy7UsW4BwPvAvd38kdYeZXRZZi0Ty9dBJsHJG+n151hbMh6bjitScLjPbzt1fBohlEh4G3A58pLxNE5E+th8X9PHuMPsjsG5FsH39SvhLQ/D84DuD42qcBi0lzsyeIMuAlruPKmFzpERUZqB66GcVTph+7Yo5yzL+sYv/ew50+nI9yhYgPB54L90Odx8eTXNEQnJP3ACk2uP7sMd3i/ZWqhMhUjcmAdsCL8c3uPsqMzuYYKqWiFQaMzh2efD8jQUwJ2macXKG4ckfBBmINUqDlhLz2djXeJ+VXINwfembI6VQijIDuh8qDpWECC9Xv5YtoJf871no9OV6lSHCAu7+pruvN7OhZjYq9hiS6fh0zOwIM1tmZs/Gpmil7j/NzB6PPR4xsz3Dnit16sETgnpD6YKDp3rwKHJw8JKZT9CxphMnUaNg1qKOor2HiFQGd/+ruy9J7ffcfY27/6jc7RORHLZsT3wW2O7wvvumDQ4+Pzx3Q3naJlIC7v6Cu78AjHH3b7r7E7HHJGBsudtXy2Yt6mDMlLkMn3QHY6bMLem9wsSxI2luauyzrZhlBnQ/VDxR/6zqSaaAnkHWf0/9DLLLGCA0syYzuxJYBdwA/B5YHg/WmdnobBc2s0bgamAcsCtwipntmnLYCuCgWLr7D4CpeZwr9eTPFjxevLnv9hFfTtwMRCBbnQgRqS0D7fdEpIIcek/w2eDoZ/tuf/TsxGeKrrXlaZtI9DY2swPjL8zsE8DGZWxPTSt3AG386FYmT9iD1pZmDGhtaWbyhD2KluGn+6HiifpnVU/SBfoMOO2Atqz/nvoZZJdtivHPgKHAh939XQAz2wz4qZn9L3AEkG2q8X7As+6+PHbuNOBY4Mn4ASn1DecBw8KeK3XggeNg1az0+07pzbsAeSGp8aoTIVJXBtrviUil2fQjiUHEx86HZ/9fYt9NmwZfkxYyE6kR5wDXm9nmsddrgLPL2J6aVgkLT0RZZkD3Q8WlkhDFMZD6u/oZZJYtQHgkMMLdN6Rmufs7ZvYl4HWC7L5sWoEXk16vAvbPcvw5wF35nmtm5wHnAbS1teVoklSFP2cI/A3ZGj73akGXLHRJedWJEKkrA+33RKSS7Xdt8HjvVZi5bWL7498NHgATXoGNtilP+0SKxN0XAnvGBrnM3d8ud5tqWa0H0HQ/JJVKgb7iyzjFGOhNvkmKc/ce4DV3n5fj2umiPGnngZrZIQQBwm/le667T3X3dndv33rrrXM0SSrWzG0TU35SfGrlPczaZVXBwUEoPDVeNQpE6spA+z0RqQYbbZMoT7JnSnnR+OeRR88rT9tEBsDMvpb8AM4Fzkl6LRHIFChrMCtLTcJiq5X7oXLWiRSpFtkyCJ80s8+7+x+SN5rZ6cBTIa69Ctgh6fUwYHXqQWY2CrgOGOfub+RzrtSATNmCwI6P3x579kGobL9kqdOJC13OfCCpyyJSdQba74lItdnt28Gjex3M2CSx/bnfBA+AzzwKW+1XnvaJ5GfTcjegHk0cO7LPTKW4ntiYY9iZS5WqFu6HCp1NJlJvLE2yRLDDrBWYCXQCCwky+PYFmoHj3D1ryN3MBgHPAIcBHcB84FR3X5p0TBswF/h8cj3CMOem097e7gsWLMh2iFSCJd+FpT9Mv+/4t9hr8mOs6ezqt6u1pZmHJx2a8/KpHQAEKanpftPDXlNEMjOzhe7eXu52DNRA+72oqG8rnkJq0UodWv47mPeF9PsiWhRNKlOt9G+Vqpb6t+T+pcFsQ3Awme47ymfMlLlpE0b0M5F6lal/y5hBGLsR2t/MDgV2I4ix3OXufwvzhu7ebWYXAHOARuB6d19qZufH9l8LfA/YErjGggUnumPThdOem8f3W7Vq+uYlS7Ygp3oQ2PvBP+js6k17SMeazlD/PummEzv9g4TVmBovItEZaL8nuZWzj1P2gIS201nBo+d9mL5R333xzzKfvBV2GF/qlomEYmbDgF8BYwg+/j4EXOjuq8rasBqWXAtt+KQ70h5TKzUJq1Gt14kUKZZsU4wBcPe5BFl+eXP3O4E7U7Zdm/T8XILaGKHOrXU1efPy+mNwT4a1aY57GZqDIuHfmfUEf5q3MuulDEL9+2T6Q+8Eo0TxG9NDdt6aK+Ys4+Lpi2svGCsiBRtIvyeZlbuPC7vKZE0P1El+GockMgaX/RoWfiWx78HjEs9P6QXLMggaEf2uShY3AH8GToi9Pj227fCytaiOaFGPyqOfiUg42RYpkRIrdCGNKAy4iOuMzYJR9jTBwVm7rGLMyr8x/PIFjJkyN1RwEIIAX5h/n0x/6OMp5Cum/P/27j1arrJM0PjzchI0wcagYisHkKgYFEGDES84KF7aC7ZEvCFiL7wMo7PQhlFs7HEJM7YDtvaoY6sMQ3vp0RYQMGYERZegtqBIMICCYCPYkIM2aSQiEiAh7/xRVTmVc/auU/fr81ur1qnau3bVV985p979vfu7HM5JL13B+VfNMLNpM8lsQ9XJaiWpNwYd45rpPVBLYhobNM+K4yvJwjcWjHL4yk6Vc54b/1ffiuPfqhawe2Z+PjO3Vm9fAFzNsU/GZVGPceLvRGqOCcIhUtZ4qQ2t7Ze2TzrvuWV2JeKtf9hh1+EbzmPNkzew5skb5r32l5tIDjYyt96aCQCDbqhK0qQZ9PCesotH9duNDeOra6tXRsyugHzo2h33XfWXs+dBD97feaEb8G9VC/j3iDgmIqaqt2OAOxc8Sl2xeuU0px15ANPLlhBUOimcduQB9vAdIH8nUnMWHGKs/mm02u4wDsParmRuwcvuWcmbbv7Q9sfvv+BnPGTRToXzA3ZibqOvmZW2Bt1QlTSeIuJlwCepzJ97VmaePmf/EcCHgG3AVuCEzPxh3ws6AIMe3lO0yuTci0fGhvHUs+Hte/757BDkuedCtbkLn/QuWNX9noX+rWoBbwX+Hvg4lVPty6vb1Cf1cxJqOPg7kRZmgnCIFDVeahom6LqsqZPOzb+Frz22+AWO3Mghn7h2XkNw85YHCz9bJ8q6hi8UAAbdUJU0fiJiCvg0lTmeNgBXRsTazLy+7mnfBdZmZkbEgcC5wH79L23/NZOg66VmLh4ZG8ZTyxc+21FLFN65Di5+5uz2X36qcgN4zZ3wkEd05e38W1UjmXkr8KpBl2NSOB+opHFhgnCI1ALJCedcXbi/n8OwSk86v/MfYGNJZ5ejZ/sC9rKsUxFsy+woAA+6oSppLB0M3JSZNwNExNnAEcD2BGFm3lP3/F3ovBP1yGgmQdePMjR6P2PDeOprb7tHrpo9H/ranrC5bijz+Y+s/HzMS+CF3+7obfxbVZGI+BQN4kpmvruPxZkIg16AS5K6yQThkFm9cpqPXnzjUA3D2nXqHq7d/6jKzo1znrz6Nli657zXaDRculPbMrnl9MM7eo1haKhKGjvTwG11jzcA81ZqiohXA6cBjwYKv8wi4jjgOIC999676wUdlGEf3mNsGE8D62336g2Vn/f8GtYun93+2+/MDkl+1c3wsOXzDl2If6sqsa7u/n8DThlUQSZFX3ooS1KfmCAcQoO+KlwLZrtd9Qaev/Sy+U9Y8lh49e0NX6PRcOlOdeuEftgbqpJGTtGErPN6cmTm14CvRcShVOYjfHHBc84EzgRYtWrVxPQyHAbGhvEz6PMqHrbPbK/CS18Bv/nm7L61j6/83OkhcNR9Lb2sf6uaKzO/WLsfESfUP1ZvOB9o93RrqLZDvqX2mSAcQgO9Krzlbvjqw1kNsHTOvlf+Enbdt6mXqZX1Pedew4PZXtt2t6WLuW/LNofPSBoVG4C96h7vCZReTcnMH0TEEyLiUZn57z0vnTShhqq33WEXVX4+cBecVzcf4bb7Z3sVPuNTsOL4/pdN48aLS33gfKDd0a2h2g75ljpjgnBI9f2q8PePgJm1xfuObu/8YvXKaU4smU+xZtmSxWzavKVw36Z7t/DxNzx9OE7oJWlhVwL7RsRyYAY4Cji6/gkR8UTgV9VFSg4Cdgbu7HtJpQkzdL3tdt5t9vxq3bvgl38/u++qd1Vu0PY5mNSOiHgZ8ElgCjgrM0+fs/9NwF9VH94DvDMzr2nm2HE18B7KY6JbQ7Ud8i11xgThJHvwPjin5OrWi/8ZHv28jt+i0VyESxZPceqr9ufUtdcVJgn3WLZk3lX/j158I9DcFSC7l0vqp8zcGhHHAxdTaSB9LjOvi4h3VPefAbwG+IuI2AJsBt6Q2WY3a0njYdWnKret98K5u+y4r9ar8PHHwrM/3/eiabRExB+Y7Tm4NCLuru0CMjN3bXDsFPBp4CVUesRfGRFrM/P6uqfdAjw/M++KiJdTmQrjWU0eO5aGqofyAHXa7urWUG2HfEudMUE4ia7/KFz9vuJ9Xb5S3Wguwocu3gmAU1+1/7znLN4puPeBrexz8oWVM5rq9ma7idu9XNIgZOZFwEVztp1Rd/8jwEf6XS5JI2DR0tnzsAufCr+/bnbfzV+o3ACO2go7TfW7dBoBmfknHRx+MHBTZt4MEBFnA0cA25N8mXl53fN/TGUqjaaOHWdD10O5RK86T3Sj3dWtodoO+ZY6s9OgC6A+2fZg5Sr0P8X85OALv1s5Ie3BMJbVK6d5zTOmmYr5c/ffde+W7cHjtCMPYHrZEoLKsGOish/mT6BS6ybeSKPu5ZIkSUPt8J9XzsveWLDY29mLKudza5/Y/3JpnE0Dt9U93lDdVuZtQG3FnaaPjYjjImJdRKzbuHFjB8VVK2pJvJlNm0lmk3hr1s+0/XqHnH4Jy0++kPece03H7a6TXrqCJYt3vPDRzlDtbr2ONKnsQTjubvo/8JPjivd1kBBs9grUmvUznH/VTOlCJbXgcdnJL9x+/CGnX1I6L2HNQt3E7V4uSZJGXuw0e772k3fCTWfM7rvnV7NDkF/3B1j8sP6XT+Nk/tX8koVOIuIwKgnC2nxETR+bmWdSGZrMqlWrnGKjT7o5N9/cHoNl7bxW2l3dGqrtkG+pMyYIx1EmfKWkc2gX5hZspRt5UTCaa27waCaYLNRN3O7lkiRprBz82coNZhODNV+tjCy97v79eOWNH7NRrHZsAPaqe7wncPvcJ0XEgcBZwMsz885WjtXgdLPzRDPtO2i93dWtodqjMuRbGkYOMR4nvz67csJYlBysDSHuwsIjrQzfbSfZt1AwaaabuN3LJUnS2Kqd1z3v3B027/+QG7jlwFdy2d4v4mNf/0Hbwwc1ka4E9o2I5RGxM3AUsLb+CRGxN3AB8ObM/GUrx2qwytpX7XSeaKZ9Z7tLGk32IBx1jXoLvuCbsMfLuv6WrVyBarSKcc0f79/KB9b8jEtv2Mjtmzbz8CWLWTwVbHlwtrt6baGS6SaviNu9XJIkjb29XwdHJ4ecfgmX7f2iHXb98ElHwy+AGx8CR903mPJpZGTm1og4HrgYmAI+l5nXRcQ7qvvPAD4IPBL4TFTmF9+amavKjh3IB1GhooUj203ilbXvpiLYlmm7SxphJghH1cyF8P1XFu/rwWIj9VoZvttoFeOaTZu38KUf37rD48U7BbstXcyme7d0NAeFgUmSJI272zdtZp9N3wDgCQ+5je+ueOfszm33zw5JfsXPYNlTB1BCjYLMvAi4aM62M+ruvx14e7PHanh0s/PEYfvtvkPbreaNz9qLv1l9QMdllTQ4JghHzdw5Z2oOOQce9/q+FKEo6RdU5iI85PRLdgg2RcHo3ge2bl+huMyWbcnSnRex/oN/1rPPIUkaP80uoiWNk/qLt7+6fy/2ubaSLLz2qUex6073zD7xorrGe48vKEsaLt3qPHHpDcWrT5dtlzQ6TBCOgt9fDxfuX7zvjdsgSpKGPVKf9JvZtHn78F8oXrBkbjBafvKFTb2PKw5LklrRyiJa0jgpGz54yf43VP7279sIFzx6x4NqF52f8yVY/qY+llZSq4bp4lej6aaGqZySWmeCcJj94EjY8LX52591Fjzhbf0vT51a0u+Q0y+ZN9y4tmBJWTBoZl7C2vMkSWpWo0W0bKCoWaPYwF1w+OBDd5/tMXjBY+C+f5s9+EfHVG6wQ6/CUawHaRx1++JXp//bZW25ZUsXd1ROv3OkwTNBOIzuuwMu+NMdty3eFV57F8RwLTzdyoIlNc3MS+jKV5KkVrUTk6R6o9wLtenhg0f+tvJz62Y4d+mO+6q9Cq/507/h/d97xkjWgzRuunnxqxvfcWU9ljNpu5yj/N0rjZPhyjapYmvdXDHP/3+Vq7mv+/3QJQfXrJ9hp5LhzY16/61eOc1pRx7A9LIlBJWViY959t47PD7tyAMMBpKklpTFnkH1SF+zfoZDTr+E5SdfyCGnX8Ka9TMDKYea16ghPnYWLamcYx6dsOerd9j1tH/7AL948sv59YGvJNgGjHE9SEOuGxe/avHohHOu7vg7rqgtd9qRB/D7zcVzzDdTzon67pWGmD0Ih9HDHj/0E0fXrvI8mPPL2UzvP1cYliR1W1mvhkH0SLc3xGga5V6oHQ3PO/SCys9M+MqOF6RvOfBVABx7yyl8f9Mzu1lkSU0oG9Lb7MWvufGoSKvfcUVtudr89O2Uc5S/e6VxMlxd0lRq2HohFF3lAZiK4LQjKyvkDVN5JUnjr6xXwyAScvaGGE3D1gu1WbUEwMymzSSzCemWz78i4OjkkFu/y/tue/cOu76w/L9xy4GvhB8eBdvKEw2Suuukl65gyeKpHba1cvGrrN1WrxvfcZ2Uc1S/e6VxYw/CETCMvRDKruZsq/YoHLbySpImw7D0ULc3xGgahl6o7fQE7PYCPZV6eDnnXvtnAHx+n1M4bNerKjtvPadyW7wrvOh78IiVLb++pOYtuAjRAhaKO936juuknMPw3SvJBOFI6MWqjL1avWqPZUtcRVKSNPE6HRKmwei0Id6pdi8KdzshPbcePrDpbznpWStYvf9SuPwYuP0i2HI3fOugygH7/mc46OMwtXNb7yepsU4ufpXFI6j0tO/md1y75Rz0d6+kChOEI6DbJ329XL3qpJeu4MRzru5qecdRpwlaSdJwszfE6BpEL9TaeUFRI76Zi6y9SEiX1sMLLqz8nLkIvn945f6/fKZyW3EiPON/tv2ekrqvLB4N26KQwzICQJpkzkE4Aro9J0M35kVqNM/TMM8hMQxzOXZtniBJ0tAapvkQR9EwxOt+qT8vKLPQRdZO5yhry/QrKovqvf6PsM+bK9u23t2795PUFuNR5yYpJmmy2YNwBHS7F0K3eiSWXeUZ1l4TwzKXo0OwJWky2BuiPcMSr/ulGwsIDHR43qKl8Nx/rNwkDaVm4pEjnIpNWkzSZOtpgjAiXgZ8EpgCzsrM0+fs3w/4PHAQ8F8z82N1+04E3g4k8DPgLZl5Xy/LO6y6fdLX63mRhnUOiWFJzDlxvSRJ5YYlXvdLuwsIFDXmLzv5hb0qZkMmFqTRZhKs3KTFJE22niUII2IK+DTwEmADcGVErM3M6+ue9jvg3cDqOcdOV7c/JTM3R8S5wFHAF3pV3mHXzV4I/ejhN4y9JoYlMefE9ZIklRuWeN0vjRYQ2G3pYk758/3nnVMNU2N+mMoiqT0mwcpNWkzSZOvlHIQHAzdl5s2Z+QBwNnBE/RMy847MvBLYUnD8ImBJRCwClgK397CsE2VS56EYlrkRBzJPkCQNmPP3qFnDEq/7pei8oOa+LdsKt3djPuluGaaySGqPSbBykxaTNNl6OcR4Grit7vEG4FnNHJiZMxHxMeBWYDPw7cz8dtFzI+I44DiAvffeu6MCT5Jh7OHXjlaGtAzL3IjDOgRbknrFHkZqxbDE636p/Q+859xreDBzh31lPXi61ZjvxtBgEwvS6HOEU7lJi0mabL1MEEbBtizYNv/AiN2o9DZcDmwCvhoRx2Tml+a9YOaZwJkAq1ataur1NR5abXAOU2JuXBK0ktQMhy6pFcMUr/tl9cppTjzn6sJ9RYm2bjTmu5W4N7Egjb5WkmCTNufoJMYkTa5eJgg3AHvVPd6T5ocJvxi4JTM3AkTEBcBzgXkJQk2udhqcJuYkqf/sYaRWTVq8XrN+hp0i5vUghOJEW1FjPoDD9tu96fds9TyqLClg7xpp9DWbBJvUEQGTFpM0uXqZILwS2DcilgMzVBYZObrJY28Fnh0RS6kMMX4RsK4npdTAtXsVqtkG56Rd5ZKkYTOKPYy6ETuMP2pGrcFdlBwsS7StXjnNun/9HV/+8a3bh+ckcP5VM6x63CO6eh5VX8ZGSQH/1qXR1kwSrN8jAoyjUn/1LEGYmVsj4njgYmAK+FxmXhcR76juPyMiHkMl8bcrsC0iTqCycvEVEXEe8FNgK7Ce6jBi9c4gvoA7uQrVTINzUq9ySdIwGbUeRt2IHcYfNauowQ0wFdFwEblLb9g4b+6eVhrqrSTuF0oK2LtGmgydjghopb1pHJX6r5erGJOZF2XmkzLzCZn54eq2MzLzjOr932bmnpm5a2Yuq96/u7rvlMzcLzOfmplvzsz7e1nWSVf7Ap7ZtJlk9gu416tMdrLyXTOrAbuyniQN3uqV05x25AFML1tCANPLljRMfAxaN2KH8aeYq1nvaM36mcIkHcC2zIb/I5021IvOo4LKOeDc343TBEiCzlb0bbW9aRyV+q+XQ4w1QgY1gXwnJ5zNDGnxhFaShsMo9TDqRuww/sw3qb1BynrM1OqjzB7LljTsbdPp0P3aMOWvXHHb9uHNtR6Jc383ozhNgKTu62REQKvtTeOo1H897UGo0TGIL+DahNxFWjm5vezkF3LL6Ydz2ckvnBdcOrnKJUmaTN2IHcaf+SaxN0ijHjNlQ4uh0uA+bL/dG/a2aWYkxUJlO/+qmcK5D2HH302n7yVpPHQyIqDV9qZxVOo/E4QC2vsC7mSYUDsTcrfDE1pJ/RQRL4uIGyPipog4uWD/myLi2urt8oh42iDKqca6ETsmMf4sdF4wib1BGiVFG33u0448gEtv2NgwodpqQ33u7+fUtdeVJihramUctWkC1J4mYth+EfGjiLg/It47Z9+JEXFdRPw8Ir4SEQ/tX8nVTwt10CjTantzEuOoNGgOMRbQenfxTocJtTshd6tcWU9Sv0TEFPBp4CXABuDKiFibmdfXPe0W4PmZeVdEvJzKAlzP6n9p1Ug3Yscox592Fi1r5rxgEoepNkqKltXH9LIlrF45zYnnXL3gazY7dL/o99OM+t/NKE0ToNY1GcN+B7wbWD3n2Onq9qdk5uaIOBc4CvhCP8qu5gx6ReBW25ujHEelUWWCUEDrX8CdzllYdsK80ITc7fCEVlKfHAzclJk3A0TE2cARwPbGVWZeXvf8HwN79rWEalo3Yke7rzHIRly7FwCbOS8YtdWsu6FRUnSh+uhmQrXRcOYy4/670TzNxLA7gDsi4vCC4xcBSyJiC7AUuL33RVazhmEO2HYSfrbjpP4yQajtWvkC7nSY0CT2IpA09qaB2+oeb6Bx78C3Ad8s2hERxwHHAey9997dKp9GwKAbce1eAGzmvGASe4M0SgIuVB/dTKg2e34WVBYqmZ6A343maTWGbZeZMxHxMeBWYDPw7cz8dveLqHYNakHKuUz4ScPNBKHa0mmCbxJ7EUgae0WrLhXO/h8Rh1FJED6vaH9mnkll+DGrVq0qXkFAY2nQjbh2LwA2e14waY3DhZKAjeqjmwnVst/PbksXs3TnRYWvX5uzcFKSuWo+hs07MGI3Kr0NlwObgK9GxDGZ+aWC53oBrEta6W3erTlg+9XDfdDDoaVJZYJQbek0wTeJvQgkjb0NwF51j/ekYIhVRBwInAW8PDPv7FPZNCIGvZBHuxcAvfBXrpOkaP2xtQbziedc3dJ505r1M9z7wNZ525csnuKUP9+/8DUG3ZNVA9FUDCvxYuCWzNwIEBEXAM8F5iUIvQDWHa3+j3Zj9Fa/vhf8/pEGx1WM1ZZurGbX7gpYkjSkrgT2jYjlEbEzlQna19Y/ISL2Bi4A3pyZvxxAGdWEhVbj7aVWV3nstnZXjXSV296qNZhnNm0mmW0wL/S3WTvurnu37LB92ZLFDX8/jXqyamwtGMMauBV4dkQsjYgAXgT8okflFK3/j3ZjReB+fS/4/SMNjj0I1bZJGyYkSY1k5taIOB5a4A6zAAARU0lEQVS4GJgCPpeZ10XEO6r7zwA+CDwS+EylDcXWzFw1qDJrvoV6LvR62NNCPfF6/f6d9PD3vKB32h16XrY4yS4PWdTxnJIaL83EsIh4DLAO2BXYFhEnUFm5+IqIOA/4KbAVWE+1l6B6o9X/0W6M3urX94LfP9LgmCCUJKlLMvMi4KI5286ou/924O39Lpeat1DPhVaGPbWTzGvUiOvXsCsTfcOn3QZzr+eU1HhpIob9lsrQ46JjTwFO6WkBtV07/6Odfrf363vB7x9pcBxiLEmSVNUoodLKsKd2h4RC+RQcDruaXO0OPW/3uG4MR5TUO4P4H+3Xe/r9Iw2OCUJJkqSqRgmVVnpjlSXz3nPuNW3Pbeiwq8nVboPZOSWl8TSI/9F+vaffP9LgOMRYkiSpqtEcgB+9+Mamhj2tWT9T+DyAB7OyaGc7w4PLhl0tW7q4qeM1utqdP8w5JaXxNYj/0X69p98/0mCYIJQkSarz0MU7bU8QLluymFNftf/2hkqjBURgdmhxM5pZZKLeSS9dwUnnXcOWB3OH7ffct5U162dsTI25dhvMNrQlSVIzHGIsSZLEbHLvrnu3bN92/9Zt2+83M+ypbNXYMq0MD169cppddp5/bXfLtmx56PKa9TMccvolbQ93liRJ0nixB6EkSRKNFwGpJQEX6o3V6nyAO0W01Pvv95u3FG5vZehyv1ZDliRJ0uiwB6EkSRprzfaW68YiIGWLnCxbsnjeYhFQSew1u7pxo9evt9DKxq6GLEmSpLlMEEqSpLFV6y03s2kzyWxvuaKEXKMVjJtVtmrsqa/an9OOPICpiHnHtJKcK3r9Io2Smq6GLEmSpLlMEEqSpLHVSm+5suRe/SIkC2k0T+HqldNsyyw8rtnk3NzXL0o4QuOkZjcSoZIkSRovzkEoSZLGViu95Wrz73304hu5fdNm9li2hJNeuqLlefkazVO4x7IlzBS8dyvJufrXnzufICyc1DzppStaPkaSJEnjzQShJEkaW60m5BZahKRT3U7OtZPU7FYiVJIkSePDBKEkSRpbw9ZbrhfJuXaSmr1OhEqSJGm0mCCUJEljaxh7y3UrObdm/cxQfS5JkiSNLhOEkiRprI1jb7m5cw/WVmcGxu6zSpIkqfdcxViSJGnEtLI6syRJkrQQE4SSJEkjppXVmSVJkqSFmCCUJEkaMWWrMJdtlyRJkhoxQShJkjRiTnrpCpYsntph2yBXZ5YkSdJo62mCMCJeFhE3RsRNEXFywf79IuJHEXF/RLx3zr5lEXFeRNwQEb+IiOf0sqySJEmjYvXKaU478gCmly0hgOllSzjtyANcoESSJElt6dkqxhExBXwaeAmwAbgyItZm5vV1T/sd8G5gdcFLfBL4Vma+NiJ2Bpb2qqySJEmjZhxXZ5YkSdJg9LIH4cHATZl5c2Y+AJwNHFH/hMy8IzOvBLbUb4+IXYFDgX+oPu+BzNzUw7JKkiRJkiRJE6mXCcJp4La6xxuq25rxeGAj8PmIWB8RZ0XELkVPjIjjImJdRKzbuHFjZyWWJEmSJEmSJkwvE4RRsC2bPHYRcBDw2cxcCfwRmDeHIUBmnpmZqzJz1e67795eSSVJkiRJkqQJ1csE4QZgr7rHewK3t3Dshsy8ovr4PCoJQ0mSJEmSJEld1MsE4ZXAvhGxvLrIyFHA2mYOzMzfArdFxIrqphcB1zc4RJIkSZLUpoh4WUTcGBE3RcS80VsRsV9E/Cgi7o+I987ZtywizouIGyLiFxHxnP6VXJLUDT1bxTgzt0bE8cDFwBTwucy8LiLeUd1/RkQ8BlgH7Apsi4gTgKdk5t3Au4AvV5OLNwNv6VVZJUmSJGlSRcQU8GngJVRGc10ZEWszs76Txu+AdwOrC17ik8C3MvO11fbb0l6XWZLUXT1LEAJk5kXARXO2nVF3/7dUhh4XHXs1sKqX5ZMkSZIkcTBwU2beDBARZwNHUDeKKzPvAO6IiMPrD4yIXYFDgWOrz3sAeKA/xZYkdUsvhxhLkiRJkobfNHBb3eMN1W3NeDywEfh8RKyPiLMiYpeiJ0bEcRGxLiLWbdy4sbMSS5K6ygShJEmSJE22KNiWTR67iMqCkp/NzJXAH4F5cxgCZOaZmbkqM1ftvvvu7ZVUktQTJgglSZIkabJtAPaqe7wncHsLx27IzCuqj8+jkjCUJI0QE4SSJEmSNNmuBPaNiOXVRUaOAtY2c2B1XvnbImJFddOLqJu7UJI0GiKz2Z7jwy8iNgL/OuhyFHgU8O+DLsQQsl7KWTfFrJdyg6ybx2Wm44R6xNg2kqybYtZLOeumnPGtTyLiFcAngCngc5n54Yh4B1QWmoyIxwDrgF2BbcA9wFMy8+6IeDpwFrAzcDPwlsy8a4H3M76NFuulnHVTzropNuh6KYxvY5UgHFYRsS4zXZF5DuulnHVTzHopZ92o3/ybK2fdFLNeylk35awb9Zt/c8Wsl3LWTTnrptiw1otDjCVJkiRJkqQJZoJQkiRJkiRJmmAmCPvjzEEXYEhZL+Wsm2LWSznrRv3m31w566aY9VLOuiln3ajf/JsrZr2Us27KWTfFhrJenINQkiRJkiRJmmD2IJQkSZIkSZImmAlCSZIkSZIkaYKZIOxARLwsIm6MiJsi4uSC/W+KiGurt8sj4mnV7XtFxKUR8YuIuC4i/rL/pe+tduumbv9URKyPiG/0r9S910m9RMSyiDgvIm6o/u08p7+l760O6+bE6v/SzyPiKxHx0P6WvneaqJcjqnVydUSsi4jnNXusVMb4VszYVs74VszYVs74pn4ztpUzvpUzvhUzvpUb6fiWmd7auAFTwK+AxwM7A9cAT5nznOcCu1Xvvxy4onr/scBB1ft/Avxy7rGjfOukbur2/xfgn4BvDPrzDEu9AF8E3l69vzOwbNCfaRjqBpgGbgGWVB+fCxw76M/Ux3p5GLPzyR4I3NDssd68Fd2Mb92vl7r9YxfbulE34xrfjG0d143xzVvXbsa23tRN3X7jW0HdGN+Mb6MW3+xB2L6DgZsy8+bMfAA4Gzii/gmZeXlm3lV9+GNgz+r232TmT6v3/wD8gso/yrhou24AImJP4HDgrD6Vt1/arpeI2BU4FPiH6vMeyMxNfSt573X0NwMsApZExCJgKXB7H8rcD83Uyz1ZjSjALkA2e6xUwvhWzNhWzvhWzNhWzvimfjO2lTO+lTO+FTO+lRvp+GaCsH3TwG11jzfQOFC8Dfjm3I0RsQ+wEriii2UbtE7r5hPA+4Bt3S/aQHVSL48HNgKfr3bfPysidulNMQei7brJzBngY8CtwG+A32fmt3tUzn5rql4i4tURcQNwIfDWVo6VChjfihnbyhnfihnbyhnf1G/GtnLGt3LGt2LGt3IjHd9MELYvCrZlwTYi4jAq/xR/NWf7w4DzgRMy8+6ul3Bw2q6biHglcEdmXtW74g1MJ38zi4CDgM9m5krgj8A4zbnTyd/MblSurCwH9gB2iYhjelTOfmuqXjLza5m5H7Aa+FArx0oFjG/FjG3ljG/FjG3ljG/qN2NbOeNbOeNbMeNbuZGObyYI27cB2Kvu8Z4UdI2NiAOpdLc+IjPvrNu+mEqA+XJmXtDjsvZbJ3VzCPCqiPg1lS61L4yIL/W2uH3TSb1sADZkZu1q5XlUAs646KRuXgzckpkbM3MLcAGVOS/GQVP1UpOZPwCeEBGPavVYqY7xrZixrZzxrZixrZzxTf1mbCtnfCtnfCtmfCs32vEth2Aix1G8UbkicDOVzHdtAsn95zxnb+Am4Llztgfwj8AnBv05hq1u5jznBYzRRLed1gvwz8CK6v1TgY8O+jMNQ90AzwKuozJ/RVCZDPhdg/5MfayXJzI7ye1BwEy1HhY81pu3opvxrfv1Muc5YxXbulE34xrfjG0d143xzVvXbsa23tTNnOcY3+Yfb3wzvo1UfFuE2pKZWyPieOBiKqvNfC4zr4uId1T3nwF8EHgk8JmIANiamauoXGl5M/CziLi6+pJ/nZkX9ftz9EKHdTO2ulAv7wK+HBE7U/nieEu/P0OvdFI3mXlFRJwH/BTYCqwHzhzE5+i2JuvlNcBfRMQWYDPwhqxEm8JjB/JBNFKMb8WMbeWMb8WMbeWMb+o3Y1s541s541sx41u5UY9vtaylJEmSJEmSpAnkHISSJEmSJEnSBDNBKEmSJEmSJE0wE4SSJEmSJEnSBDNBKEmSJEmSJE0wE4SSJEmSJEnSBDNBqJETEXtGxNcj4l8i4lcR8cnq0vG9fM9jI+Lve/kede91akTMRMTVdbdlbbzOCyLiGx2U46/nPL683deSJDVmbGv6dYxtkjRCjG9Nv47xTQNnglAjJSICuABYk5n7Ak8CHgZ8eKAF676PZ+bT626bBlCGHYJMZj53AGWQpLFnbOsrY5sk9Ynxra+Mb+qYCUKNmhcC92Xm5wEy80HgROCtEbG0erXo6xHxrYi4MSJOqR0YEcdExE+qV3X+d0RMVbffExEfjohrIuLHEfGnzRYmIt4YET+LiJ9HxEea2H5PRPxdRPw0Ir4bEbu38F5XRMT+dY+/FxHPiIiDI+LyiFhf/bmi4NhTI+K9dY9/HhH7VO+viYirIuK6iDiuuu10YEm1rr5cK3v1Z0TER6uv8bOIeEN1+wuqZTovIm6IiC9XTwokSY0Z22YfG9skaXwY32YfG9809EwQatTsD1xVvyEz7wZuBZ5Y3XQw8Cbg6cDrImJVRDwZeANwSGY+HXiw+hyAXYAfZ+bTgB8A/7GZgkTEHsBHqAS+pwPPjIjVZdvr3uunmXkQ8H3glPmvDMCJMdtF/dLqtrOB11ff+7HAHpl5FXADcGhmrgQ+CPyPZspf562Z+QxgFfDuiHhkZp4MbK5eAXvTnOcfWf1cTwNeDHy0Wh6AlcAJwFOAxwOHtFgWSZpExjaMbZI0hoxvGN80OhYNugBSiwLIBbZ/JzPvBIiIC4DnAVuBZwBXVi+MLAHuqD7/AaA238NVwEuaLMszge9l5sbqe30ZOLRajqLta4BtwDnV479Epct9kY9n5sfmbDsX+A6VwPR64KvV7Q8HvhgR+1bfe3GT5a95d0S8unp/L2Bf4M4Gz38e8JXqFcB/i4jvU6mLu4GfZOYGgIi4GtgH+GGL5ZGkSWNsM7ZJ0jgyvhnfNEJMEGrUXAe8pn5DROxK5cvxV1QCydwglFSC0Bcz8/0Fr7klM2vHPEjz/xdlXbBb6ZpdFDCLn5g5ExF3RsSBVK6o/afqrg8Bl2bmq6tdz79XcPhWduwx/FCodC2nciXpOZl5b0R8r7avgUaf7/66+63UpSRNMmObsU2SxpHxzfimEeIQY42a7wJLI+IvAKIyF8XfAV/IzHurz3lJRDwiIpYAq4HLqse9NiIeXT3uERHxuA7LcgXw/Ih4VLUcb6TS9bxsO1T+515bvX80rV+hORt4H/DwzPxZddvDgZnq/WNLjvs1cBBARBwELK879q5qgNkPeHbdMVsiouiK1g+AN0TEVFTm4TgU+EmLn0OSNMvYZmyTpHFkfDO+aYSYINRIqV4tejWV+Sn+BfglcB87rtr0Q+D/AlcD52fmusy8HvgA8O2IuJZKd+/H0ppjI2JD7QZMAe8HLgWuoTI/xdcz8zdF26uv8Udg/4i4iso8F/+95L3q57G4unp1CeA84CgqXdZr/hY4LSIuq5apyPnAI6pdx99Jpd4AvgUsqtbJh4Af1x1zJnBttZt9va8B11Y/2yXA+zLztyXvK0lagLHN2CZJ48j4ZnzTaInZ3rnS6IuIY4FVmXn8oMtSJCLuycyHDbockqTRYWyTJI0j45s0XOxBKEmSJEmSJE0wexBKkiRJkiRJE8wehJIkSZIkSdIEM0EoSZIkSZIkTTAThJIkSZIkSdIEM0EoSZIkSZIkTTAThJIkSZIkSdIE+/+wNlmZ0scOMwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1296x288 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib\n",
    "# avoid type 3 fonts\n",
    "matplotlib.rcParams['pdf.fonttype'] = 42\n",
    "matplotlib.rcParams['ps.fonttype'] = 42\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "from scipy.stats import kendalltau\n",
    "\n",
    "fig, ((ax1, ax2, ax3)) = plt.subplots(nrows=1, ncols=3)    \n",
    "fig.set_figwidth(18)\n",
    "\n",
    "\n",
    "\n",
    "cmap = plt.cm.Spectral\n",
    "ax1.scatter(ranking_open, ranking_q1, label = ranking_groups)\n",
    "# ax1.set_xlim([0,1])\n",
    "# ax1.set_ylim([0,1])\n",
    "ax1.set_xlabel('Open Loop Evaluation')\n",
    "ax1.set_ylabel('Q1 Evaluation')\n",
    "m, b = np.polyfit(ranking_open, ranking_q1, 1)\n",
    "ax1.plot(ranking_open, m*ranking_open + b, color='orange')\n",
    "tau_open_q1, _ = kendalltau(ranking_open, ranking_q1)\n",
    "ax1.text(0.216, 0.258, r'$\\tau=$'+ \"%.2f\" % tau_open_q1, fontsize=14, bbox={'facecolor':'red', 'alpha':0.5, 'pad':10})\n",
    "\n",
    "ax2.scatter(ranking_open, ranking_q2, label = ranking_groups)\n",
    "# ax2.set_ylim([0,1])\n",
    "ax2.set_xlabel('Open Loop Evaluation')\n",
    "ax2.set_ylabel('Q2 Evaluation')\n",
    "m, b = np.polyfit(ranking_open, ranking_q2, 1)\n",
    "ax2.plot(ranking_open, m*ranking_open + b, color='orange')\n",
    "tau_open_q2, _ = kendalltau(ranking_open, ranking_q2)\n",
    "ax2.text(0.216, 0.63, r'$\\tau=$'+ \"%.2f\" % tau_open_q2, fontsize=14, bbox={'facecolor':'red', 'alpha':0.5, 'pad':10})\n",
    "\n",
    "\n",
    "ax3.scatter(ranking_open, ranking_close, label = ranking_groups)\n",
    "# ax3.set_ylim([0,1])\n",
    "ax3.set_xlabel('Open Loop Evaluation')\n",
    "ax3.set_ylabel('Holdout Evaluation')\n",
    "m, b = np.polyfit(ranking_open, ranking_close, 1)\n",
    "ax3.plot(ranking_open, m*ranking_open + b, color='orange')\n",
    "tau_open_closed, _ = kendalltau(ranking_open, ranking_close)\n",
    "ax3.text(0.216, 0.28, r'$\\tau=$'+ \"%.2f\" % tau_open_closed, fontsize=14, bbox={'facecolor':'red', 'alpha':0.5, 'pad':10})\n",
    "\n",
    "\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.subplots_adjust(wspace = 0.25)\n",
    "\n",
    "plt.show() \n",
    "\n",
    "fig.savefig('scatter_coat.pdf', format='pdf');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compare all models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>METRIC</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>OPEN</th>\n",
       "      <th>CLOSED</th>\n",
       "      <th>IPS</th>\n",
       "      <th>UNBIASED</th>\n",
       "      <th>Q1</th>\n",
       "      <th>Q2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>624</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>GA</td>\n",
       "      <td>0.233</td>\n",
       "      <td>0.178</td>\n",
       "      <td>0.161</td>\n",
       "      <td>0.175</td>\n",
       "      <td>0.176</td>\n",
       "      <td>0.160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>625</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>MPOP</td>\n",
       "      <td>0.242</td>\n",
       "      <td>0.246</td>\n",
       "      <td>0.234</td>\n",
       "      <td>0.227</td>\n",
       "      <td>0.195</td>\n",
       "      <td>0.666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>626</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>BaselineOnly</td>\n",
       "      <td>0.261</td>\n",
       "      <td>0.222</td>\n",
       "      <td>0.212</td>\n",
       "      <td>0.216</td>\n",
       "      <td>0.217</td>\n",
       "      <td>0.206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>627</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>MLP</td>\n",
       "      <td>0.249</td>\n",
       "      <td>0.243</td>\n",
       "      <td>0.232</td>\n",
       "      <td>0.229</td>\n",
       "      <td>0.205</td>\n",
       "      <td>0.550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>628</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>MF10</td>\n",
       "      <td>0.236</td>\n",
       "      <td>0.180</td>\n",
       "      <td>0.167</td>\n",
       "      <td>0.175</td>\n",
       "      <td>0.174</td>\n",
       "      <td>0.184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>723</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>NeuMF60</td>\n",
       "      <td>0.253</td>\n",
       "      <td>0.267</td>\n",
       "      <td>0.246</td>\n",
       "      <td>0.253</td>\n",
       "      <td>0.235</td>\n",
       "      <td>0.498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>724</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>NeuMF70</td>\n",
       "      <td>0.249</td>\n",
       "      <td>0.260</td>\n",
       "      <td>0.245</td>\n",
       "      <td>0.248</td>\n",
       "      <td>0.232</td>\n",
       "      <td>0.456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>725</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>NeuMF80</td>\n",
       "      <td>0.251</td>\n",
       "      <td>0.266</td>\n",
       "      <td>0.253</td>\n",
       "      <td>0.253</td>\n",
       "      <td>0.237</td>\n",
       "      <td>0.465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>726</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>NeuMF90</td>\n",
       "      <td>0.247</td>\n",
       "      <td>0.264</td>\n",
       "      <td>0.250</td>\n",
       "      <td>0.251</td>\n",
       "      <td>0.236</td>\n",
       "      <td>0.451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>727</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>NeuMF100</td>\n",
       "      <td>0.248</td>\n",
       "      <td>0.260</td>\n",
       "      <td>0.247</td>\n",
       "      <td>0.246</td>\n",
       "      <td>0.227</td>\n",
       "      <td>0.503</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>104 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      METRIC         MODEL   OPEN CLOSED    IPS UNBIASED     Q1     Q2\n",
       "624  NDCG@-1            GA  0.233  0.178  0.161    0.175  0.176  0.160\n",
       "625  NDCG@-1          MPOP  0.242  0.246  0.234    0.227  0.195  0.666\n",
       "626  NDCG@-1  BaselineOnly  0.261  0.222  0.212    0.216  0.217  0.206\n",
       "627  NDCG@-1           MLP  0.249  0.243  0.232    0.229  0.205  0.550\n",
       "628  NDCG@-1          MF10  0.236  0.180  0.167    0.175  0.174  0.184\n",
       "..       ...           ...    ...    ...    ...      ...    ...    ...\n",
       "723  NDCG@-1       NeuMF60  0.253  0.267  0.246    0.253  0.235  0.498\n",
       "724  NDCG@-1       NeuMF70  0.249  0.260  0.245    0.248  0.232  0.456\n",
       "725  NDCG@-1       NeuMF80  0.251  0.266  0.253    0.253  0.237  0.465\n",
       "726  NDCG@-1       NeuMF90  0.247  0.264  0.250    0.251  0.236  0.451\n",
       "727  NDCG@-1      NeuMF100  0.248  0.260  0.247    0.246  0.227  0.503\n",
       "\n",
       "[104 rows x 8 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pickle\n",
    "import qgrid\n",
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "from scipy.stats import kendalltau, ttest_rel\n",
    "from utils import natural_keys\n",
    "\n",
    "\n",
    "# load results\n",
    "with open('./data/exp_open_coats.pkl', 'rb') as exp_file:\n",
    "    res_open = pickle.load(exp_file)\n",
    "\n",
    "with open('./data/exp_stra_coats.pkl', 'rb') as exp_file:\n",
    "    res_stra = pickle.load(exp_file)\n",
    "\n",
    "\n",
    "    \n",
    "def get_metric_value(results, model_name, metric, user_base=False):\n",
    "\n",
    "    for result in results:\n",
    "        if result.model_name == model_name:\n",
    "            if user_base:\n",
    "                return result.metric_user_results[metric]\n",
    "            else:\n",
    "                return result.metric_avg_results[metric]\n",
    "\n",
    "    \n",
    "METRICS = sorted([m for m in res_open[0].metric_avg_results.keys() if '(s)' not in m], \n",
    "                 key=natural_keys)\n",
    "\n",
    "MODELS  = [res.model_name for res in res_open]\n",
    "\n",
    "\n",
    "idx = 0\n",
    "\n",
    "df = pd.DataFrame(columns=['METRIC', 'MODEL', 'OPEN', 'CLOSED', 'IPS', 'UNBIASED', 'Q1', 'Q2'])\n",
    "\n",
    "for metric in METRICS:\n",
    "        \n",
    "    for model in MODELS:\n",
    "        \n",
    "        open_loop = get_metric_value(res_open, model, metric)\n",
    "        closed = get_metric_value([res_array[0] for res_array in res_stra], model, metric)\n",
    "        ips = get_metric_value([res_array[1] for res_array in res_stra], model, metric)\n",
    "        q1 = get_metric_value([res_array[3] for res_array in res_stra], model, metric)\n",
    "        q2 = get_metric_value([res_array[4] for res_array in res_stra], model, metric)\n",
    "        unbiased = get_metric_value([res_array[5] for res_array in res_stra], model, metric)\n",
    "        \n",
    "        df.loc[idx] = [metric, model, \n",
    "                       '%.3f' %(open_loop),\n",
    "                       '%.3f' %(closed), \n",
    "                       '%.3f' %(ips),\n",
    "                       '%.3f' %(unbiased), \n",
    "                       '%.3f' %(q1), \n",
    "                       '%.3f' %(q2)]\n",
    "        idx += 1\n",
    "    \n",
    "\n",
    "\n",
    "display(df[df.METRIC == 'NDCG@-1'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Open Loop (Random) Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 110/290 [00:00<00:00, 1099.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rating_threshold = 4.0\n",
      "exclude_unknowns = False\n",
      "---\n",
      "Training data:\n",
      "Number of users = 290\n",
      "Number of items = 300\n",
      "Number of ratings = 6960\n",
      "Max rating = 5.0\n",
      "Min rating = 1.0\n",
      "Global mean = 2.6\n",
      "---\n",
      "Test data:\n",
      "Number of users = 290\n",
      "Number of items = 300\n",
      "Number of ratings = 4640\n",
      "Number of unknown users = 0\n",
      "Number of unknown items = 0\n",
      "---\n",
      "Total users = 290\n",
      "Total items = 300\n",
      "\n",
      "[MPOP] Training started!\n",
      "\n",
      "[MPOP] Evaluation started!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 290/290 [00:00<00:00, 1125.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[WMF32] Training started!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▍      | 100/290 [00:00<00:00, 555.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[WMF32] Evaluation started!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 290/290 [00:00<00:00, 515.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "TEST:\n",
      "...\n",
      "      | NDCG@-1 | NDCG@10 | NDCG@100 | NDCG@20 | NDCG@30 | NDCG@5 | Train (s) | Test (s)\n",
      "----- + ------- + ------- + -------- + ------- + ------- + ------ + --------- + --------\n",
      "MPOP  |  0.2420 |  0.0354 |   0.1248 |  0.0560 |  0.0672 | 0.0256 |    0.0013 |   0.2649\n",
      "WMF32 |  0.2473 |  0.0375 |   0.1373 |  0.0554 |  0.0676 | 0.0323 |    4.4464 |   0.5658\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from cornac.eval_methods.base_method import BaseMethod\n",
    "from cornac.experiment.experiment import Experiment\n",
    "\n",
    "from utils import get_models, get_metrics\n",
    "from datasets import yahoo_music, coats\n",
    "\n",
    "\n",
    "# load the closed/open loop datasets\n",
    "ds_closed = coats.load_feedback(variant='closed_loop')\n",
    "ds_open = coats.load_feedback(variant='open_loop')\n",
    "\n",
    "\n",
    "# train on closed-loop dataset and evaluate on open loop (random) dataset\n",
    "eval_method = BaseMethod.from_splits(train_data=ds_closed, \n",
    "                                     test_data=ds_open, \n",
    "                                     rating_threshold=4.0,\n",
    "                                     verbose=True)\n",
    "\n",
    "# run the experiment: use variant='large' to produce for all models\n",
    "exp_open = Experiment(eval_method=eval_method,\n",
    "                      models=get_models(variant='small'),\n",
    "                      metrics=get_metrics(variant='small'),\n",
    "                      verbose=True)\n",
    "\n",
    "exp_open.run()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Propensity-based Stratified Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Calculating best minimal value for power law fit\n",
      " 43%|████▎     | 125/289 [00:00<00:00, 1245.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rating_threshold = 4.0\n",
      "exclude_unknowns = True\n",
      "Powerlaw exponential estimates: 4.014793, min=28\n",
      "---\n",
      "Training data:\n",
      "Number of users = 290\n",
      "Number of items = 300\n",
      "Number of ratings = 5568\n",
      "Max rating = 5.0\n",
      "Min rating = 1.0\n",
      "Global mean = 2.6\n",
      "---\n",
      "Test data (Q0):\n",
      "Number of users = 289\n",
      "Number of items = 293\n",
      "Number of ratings = 1392\n",
      "Max rating = 5.0\n",
      "Min rating = 1.0\n",
      "Global mean = 2.6\n",
      "Number of unknown users = 0\n",
      "Number of unknown items = 0\n",
      "---\n",
      "Test data (Q1):\n",
      "Number of users = 284\n",
      "Number of items = 288\n",
      "Number of ratings = 1307\n",
      "Max rating = 5.0\n",
      "Min rating = 1.0\n",
      "Global mean = 2.5\n",
      "Number of unknown users = 0\n",
      "Number of unknown items = 0\n",
      "---\n",
      "Test data (Q2):\n",
      "Number of users = 59\n",
      "Number of items = 5\n",
      "Number of ratings = 85\n",
      "Max rating = 5.0\n",
      "Min rating = 1.0\n",
      "Global mean = 2.9\n",
      "Number of unknown users = 0\n",
      "Number of unknown items = 0\n",
      "---\n",
      "Total users = 290\n",
      "Total items = 300\n",
      "\n",
      "[MPOP] Training started!\n",
      "\n",
      "[MPOP] Evaluation started!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 289/289 [00:00<00:00, 1275.25it/s]\n",
      " 36%|███▋      | 105/289 [00:00<00:00, 1049.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[MPOP] IPS Evaluation started!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 289/289 [00:00<00:00, 1105.79it/s]\n",
      " 37%|███▋      | 106/289 [00:00<00:00, 1053.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[MPOP] SNIPS Evaluation started!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 289/289 [00:00<00:00, 1106.18it/s]\n",
      " 46%|████▋     | 132/284 [00:00<00:00, 1311.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[MPOP] Stratified Evaluation started!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 284/284 [00:00<00:00, 1292.16it/s]\n",
      "100%|██████████| 59/59 [00:00<00:00, 1840.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[WMF32] Training started!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▍      | 100/289 [00:00<00:00, 746.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[WMF32] Evaluation started!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 289/289 [00:00<00:00, 706.38it/s]\n",
      " 35%|███▍      | 100/289 [00:00<00:00, 625.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[WMF32] IPS Evaluation started!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 289/289 [00:00<00:00, 660.60it/s]\n",
      " 35%|███▍      | 100/289 [00:00<00:00, 619.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[WMF32] SNIPS Evaluation started!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 289/289 [00:00<00:00, 661.43it/s]\n",
      " 35%|███▌      | 100/284 [00:00<00:00, 678.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[WMF32] Stratified Evaluation started!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 284/284 [00:00<00:00, 754.18it/s]\n",
      "100%|██████████| 59/59 [00:00<00:00, 882.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "TEST:\n",
      "...\n",
      "[MPOP]\n",
      "         |     NDCG@-1 |    NDCG@10 |    NDCG@100 |     NDCG@20 |     NDCG@30 | NDCG@5 |      SIZE\n",
      "-------- + ----------- + ---------- + ----------- + ----------- + ----------- + ------ + ---------\n",
      "Closed   |      0.2448 |     0.0740 |      0.1733 |      0.0941 |      0.1141 | 0.0588 | 1392.0000\n",
      "-------- + ----------- + ---------- + ----------- + ----------- + ----------- + ------ + ---------\n",
      "IPS      |      0.2318 |     0.0751 |      0.1683 |      0.0930 |      0.1103 | 0.0639 | 1392.0000\n",
      "SNIPS    | 297814.1119 | 97405.4612 | 163905.0640 | 154477.8742 | 160712.2154 | 0.5730 | 1392.0000\n",
      "-------- + ----------- + ---------- + ----------- + ----------- + ----------- + ------ + ---------\n",
      "Q1       |      0.1985 |     0.0167 |      0.1235 |      0.0376 |      0.0588 | 0.0041 | 1307.0000\n",
      "Q2       |      0.6815 |     0.6815 |      0.6815 |      0.6815 |      0.6815 | 0.6320 |   85.0000\n",
      "-------- + ----------- + ---------- + ----------- + ----------- + ----------- + ------ + ---------\n",
      "Unbiased |      0.2280 |     0.0573 |      0.1576 |      0.0769 |      0.0969 | 0.0425 | 1392.0000\n",
      "\n",
      "[WMF32]\n",
      "         |     NDCG@-1 |     NDCG@10 |    NDCG@100 |     NDCG@20 |     NDCG@30 |      NDCG@5 |      SIZE\n",
      "-------- + ----------- + ----------- + ----------- + ----------- + ----------- + ----------- + ---------\n",
      "Closed   |      0.2082 |      0.0197 |      0.1532 |      0.0328 |      0.0512 |      0.0083 | 1392.0000\n",
      "-------- + ----------- + ----------- + ----------- + ----------- + ----------- + ----------- + ---------\n",
      "IPS      |      0.1933 |      0.0197 |      0.1436 |      0.0317 |      0.0461 |      0.0090 | 1392.0000\n",
      "SNIPS    | 307673.4000 | 111487.6887 | 183076.0337 | 111487.8170 | 113523.9500 | 111487.5406 | 1392.0000\n",
      "-------- + ----------- + ----------- + ----------- + ----------- + ----------- + ----------- + ---------\n",
      "Q1       |      0.2045 |      0.0203 |      0.1488 |      0.0328 |      0.0511 |      0.0086 | 1307.0000\n",
      "Q2       |      0.1862 |      0.0000 |      0.1517 |      0.0289 |      0.0424 |      0.0000 |   85.0000\n",
      "-------- + ----------- + ----------- + ----------- + ----------- + ----------- + ----------- + ---------\n",
      "Unbiased |      0.2033 |      0.0191 |      0.1490 |      0.0325 |      0.0506 |      0.0081 | 1392.0000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from datasets import yahoo_music, coats\n",
    "from utils import get_models, get_metrics\n",
    "from eval_methods.stratified_evaluation import StratifiedEvaluation\n",
    "from experiment.experiment import STExperiment\n",
    "\n",
    "\n",
    "# load the closed loop datasets\n",
    "ds_closed = coats.load_feedback(variant='closed_loop')\n",
    "\n",
    "\n",
    "\n",
    "stra_eval_method = StratifiedEvaluation(data=ds_closed,\n",
    "                                        n_strata=2,\n",
    "                                        rating_threshold=4.0,\n",
    "                                        verbose=True)\n",
    "\n",
    "# run the experiment: use variant='large' to produce for all models\n",
    "exp_stra = STExperiment(eval_method=stra_eval_method,\n",
    "                        models=get_models(variant='small'),\n",
    "                        metrics=get_metrics(variant='small'),\n",
    "                        verbose=True)\n",
    "\n",
    "exp_stra.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}