examples/out/train-resnet50-wids.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "51f832f5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T08:07:06.177213Z",
     "iopub.status.busy": "2023-12-18T08:07:06.176924Z",
     "iopub.status.idle": "2023-12-18T08:07:08.385233Z",
     "shell.execute_reply": "2023-12-18T08:07:08.384442Z"
    },
    "papermill": {
     "duration": 2.216329,
     "end_time": "2023-12-18T08:07:08.388581",
     "exception": false,
     "start_time": "2023-12-18T08:07:06.172252",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from functools import partial\n",
    "from pprint import pprint\n",
    "import random\n",
    "from collections import deque\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from torchvision.models import resnet50\n",
    "from torch.utils.data import DataLoader\n",
    "from torch import nn, optim\n",
    "import wids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d941678f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T08:07:08.400260Z",
     "iopub.status.busy": "2023-12-18T08:07:08.399987Z",
     "iopub.status.idle": "2023-12-18T08:07:08.403830Z",
     "shell.execute_reply": "2023-12-18T08:07:08.403217Z"
    },
    "papermill": {
     "duration": 0.012929,
     "end_time": "2023-12-18T08:07:08.406995",
     "exception": false,
     "start_time": "2023-12-18T08:07:08.394066",
     "status": "completed"
    },
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# parameters\n",
    "epochs = 3\n",
    "max_steps = 100000\n",
    "batch_size = 32\n",
    "bucket = \"https://storage.googleapis.com/webdataset/fake-imagenet/\"\n",
    "num_workers = 4\n",
    "cache_dir = \"./_cache\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "457e57ee",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T08:07:08.417423Z",
     "iopub.status.busy": "2023-12-18T08:07:08.417237Z",
     "iopub.status.idle": "2023-12-18T08:07:08.420538Z",
     "shell.execute_reply": "2023-12-18T08:07:08.419930Z"
    },
    "papermill": {
     "duration": 0.012124,
     "end_time": "2023-12-18T08:07:08.423760",
     "exception": false,
     "start_time": "2023-12-18T08:07:08.411636",
     "status": "completed"
    },
    "tags": [
     "injected-parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "max_steps = 10000\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5f37f557",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T08:07:08.434615Z",
     "iopub.status.busy": "2023-12-18T08:07:08.434239Z",
     "iopub.status.idle": "2023-12-18T08:07:08.439154Z",
     "shell.execute_reply": "2023-12-18T08:07:08.438216Z"
    },
    "papermill": {
     "duration": 0.013834,
     "end_time": "2023-12-18T08:07:08.442254",
     "exception": false,
     "start_time": "2023-12-18T08:07:08.428420",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# helpers\n",
    "\n",
    "import time\n",
    "\n",
    "\n",
    "def enumerate_report(seq, delta, growth=1.0):\n",
    "    last = 0\n",
    "    count = 0\n",
    "    for count, item in enumerate(seq):\n",
    "        now = time.time()\n",
    "        if now - last > delta:\n",
    "            last = now\n",
    "            yield count, item, True\n",
    "        else:\n",
    "            yield count, item, False\n",
    "        delta *= growth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5fbdbcc1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T08:07:08.453706Z",
     "iopub.status.busy": "2023-12-18T08:07:08.453356Z",
     "iopub.status.idle": "2023-12-18T08:07:08.461374Z",
     "shell.execute_reply": "2023-12-18T08:07:08.460382Z"
    },
    "papermill": {
     "duration": 0.017358,
     "end_time": "2023-12-18T08:07:08.464442",
     "exception": false,
     "start_time": "2023-12-18T08:07:08.447084",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# The standard TorchVision transformations.\n",
    "\n",
    "transform_train = transforms.Compose(\n",
    "    [\n",
    "        transforms.RandomResizedCrop(224),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
    "    ]\n",
    ")\n",
    "\n",
    "transform_val = transforms.Compose(\n",
    "    [\n",
    "        transforms.Resize(256),\n",
    "        transforms.CenterCrop(224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
    "    ]\n",
    ")\n",
    "\n",
    "# The dataset returns dictionaries. This is a small function we transform it\n",
    "# with to get the augmented image and the label.\n",
    "\n",
    "\n",
    "def make_sample(sample, val=False):\n",
    "    image = sample[\".jpg\"]\n",
    "    label = sample[\".cls\"]\n",
    "    if val:\n",
    "        return transform_val(image), label\n",
    "    else:\n",
    "        return transform_train(image), label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9386bb8d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T08:07:08.476096Z",
     "iopub.status.busy": "2023-12-18T08:07:08.475734Z",
     "iopub.status.idle": "2023-12-18T08:07:09.246987Z",
     "shell.execute_reply": "2023-12-18T08:07:09.245651Z"
    },
    "papermill": {
     "duration": 0.781493,
     "end_time": "2023-12-18T08:07:09.250771",
     "exception": false,
     "start_time": "2023-12-18T08:07:08.469278",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "https://storage.googleapis.com/webdataset/fake-ima base: https://storage.googleapis.com/webdataset/fake-imagenet name: imagenet-train nfiles: 1282 nbytes: 31242280960 samples: 128200 cache: ./_cache\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "https://storage.googleapis.com/webdataset/fake-ima base: https://storage.googleapis.com/webdataset/fake-imagenet name: imagenet-val nfiles: 50 nbytes: 1219143680 samples: 5000 cache: ./_cache\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'.cls': 6,\n",
       " '.jpg': <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=512x512>,\n",
       " '.txt': 'a high quality color photograph of a lion',\n",
       " '__key__': '000000-000003',\n",
       " '__dataset__': None,\n",
       " '__index__': 0,\n",
       " '__shard__': 'https://storage.googleapis.com/webdataset/fake-imagenet/imagenet-train-000000.tar',\n",
       " '__shardindex__': 0}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# These are standard PyTorch datasets. Download is incremental into the cache.\n",
    "\n",
    "trainset = wids.ShardListDataset(\n",
    "    bucket+\"imagenet-train.json\", cache_dir=cache_dir, keep=True\n",
    ")\n",
    "valset = wids.ShardListDataset(\n",
    "    bucket+\"imagenet-val.json\", cache_dir=cache_dir, keep=True\n",
    ")\n",
    "\n",
    "trainset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ebb601aa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T08:07:09.264653Z",
     "iopub.status.busy": "2023-12-18T08:07:09.264255Z",
     "iopub.status.idle": "2023-12-18T08:07:09.290057Z",
     "shell.execute_reply": "2023-12-18T08:07:09.289243Z"
    },
    "papermill": {
     "duration": 0.034928,
     "end_time": "2023-12-18T08:07:09.292076",
     "exception": false,
     "start_time": "2023-12-18T08:07:09.257148",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 224, 224]) 6\n"
     ]
    }
   ],
   "source": [
    "# Next, we add the transformation to the dataset. Transformations\n",
    "# are executed in sequence. In fact, by default, there is a transformation\n",
    "# that reads and decodes images.\n",
    "\n",
    "trainset.add_transform(make_sample)\n",
    "valset.add_transform(partial(make_sample, val=True))\n",
    "\n",
    "print(trainset[0][0].shape, trainset[0][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b39e76bc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T08:07:09.299889Z",
     "iopub.status.busy": "2023-12-18T08:07:09.299498Z",
     "iopub.status.idle": "2023-12-18T08:07:09.532940Z",
     "shell.execute_reply": "2023-12-18T08:07:09.532321Z"
    },
    "papermill": {
     "duration": 0.239314,
     "end_time": "2023-12-18T08:07:09.534590",
     "exception": false,
     "start_time": "2023-12-18T08:07:09.295276",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tmb/proj/webdataset/wids/wids.py:740: UserWarning: DistributedChunkedSampler is called without distributed initialized; assuming single process\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABgsElEQVR4nO3deVxUVf8H8M+wDaAMiMimqCgK7ruIW5okmu32lGVq5vLTByu11HwyM+vJsmx5ekzrabFFSyu10lxw33BDcUElFxQ3QEUYkH3m/v7AGefOPsMMw8x83q8Xr5h7z9x77h3jfuec7zlHIgiCACIiIiIn4uHoChARERFZigEMEREROR0GMEREROR0GMAQERGR02EAQ0RERE6HAQwRERE5HQYwRERE5HQYwBAREZHT8XJ0BexFqVTi2rVrCAgIgEQicXR1iIiIyAyCIKCoqAiRkZHw8DDczuKyAcy1a9cQFRXl6GoQERGRFS5fvowmTZoY3O+yAUxAQACA6hsgk8kcXBsiIiIyh1wuR1RUlPo5bojLBjCqbiOZTMYAhoiIyMmYSv9gEi8RERE5HQYwRERE5HQYwBAREZHTYQBDRERETocBDBERETkdBjBERETkdBjAEBERkdNhAENEREROhwEMEREROR2LApglS5agY8eO6tltExISsGHDBvX+srIyJCcno2HDhqhfvz6GDx+O3Nxc0TGys7MxbNgw+Pv7IzQ0FDNmzEBVVZWozI4dO9C1a1dIpVLExMRg2bJl1l8hERERuRyLApgmTZrgvffeQ1paGg4fPoz7778fjz76KDIyMgAA06ZNw59//olffvkFO3fuxLVr1/DEE0+o369QKDBs2DBUVFRg3759+O6777Bs2TLMnTtXXSYrKwvDhg3DwIEDkZ6ejqlTp2L8+PHYtGmTjS6ZiIiInJ1EEAShJgcIDg7GBx98gCeffBKNGjXCihUr8OSTTwIAzpw5gzZt2iA1NRW9evXChg0b8NBDD+HatWsICwsDACxduhSzZs3CjRs34OPjg1mzZmH9+vU4efKk+hwjRoxAQUEBNm7caHa95HI5AgMDUVhYyLWQiIiInIS5z2+rc2AUCgV+/vln3LlzBwkJCUhLS0NlZSUSExPVZeLi4tC0aVOkpqYCAFJTU9GhQwd18AIASUlJkMvl6lac1NRU0TFUZVTHMKS8vBxyuVz0Q0REZImySgUMfa8vqajC7+lXUVhaWat1qqhS4mBWPioVSquP8VvaFew+ewOVCiWuF5basHaOY3EAc+LECdSvXx9SqRSTJk3CmjVr0LZtW+Tk5MDHxwdBQUGi8mFhYcjJyQEA5OTkiIIX1X7VPmNl5HI5SksN3/QFCxYgMDBQ/RMVFWXppRERkRu7crsEcW9sRPKKI3r3z1lzEi//nI5/Lk+zaz3KqxSi12+sPYmnvkjFm39k6JQ7fDEfVVqBzbd7s/DUF6koKqvEtYJSnMsrxiu/HMOorw/iyaWpSFiwDWmXbtv1GmqDxQFMbGws0tPTceDAAUyePBljxozBqVOn7FE3i8yePRuFhYXqn8uXLzu6SkRE5CQEQUDf97cDAP46kaO3zOqjVwEAe8/dsls9Zv56DLFzNuL8jWL1tpWHq59nKw5ki8pOX3kMTy5NxaKUv0Xb3/rzFA5m5aPDvM3o/d42fLHzvHrfscsFAIBf067Y6Qpqj8UBjI+PD2JiYtCtWzcsWLAAnTp1wqefforw8HBUVFSgoKBAVD43Nxfh4eEAgPDwcJ1RSarXpsrIZDL4+fkZrJdUKlWPjlL9EBERmcPSoEShFLBocyb2nL1psIwgCBj19QGM/fag3m6py/kl2J6ZJ9q26nB1YPHV7iyTdVh/4joA4OvdWVh//Dr+sXSf3u6hX/QGKzVKf60TajwPjFKpRHl5Obp16wZvb29s3bpVvS8zMxPZ2dlISEgAACQkJODEiRPIy7v3gaWkpEAmk6Ft27bqMprHUJVRHYOIiMjWLM1r+e3IFXy27Rye+/qAwTI3isqx++xNbM+8AXmpeLqQ0goF+i3cjrHfHsLuszd03iuRWFQdJK84gkMXb2OeVjeTKWuOXjHZnbQpIwevrDqG0gqF0XK1zaIAZvbs2di1axcuXryIEydOYPbs2dixYwdGjhyJwMBAjBs3DtOnT8f27duRlpaGsWPHIiEhAb169QIADB48GG3btsWoUaNw7NgxbNq0CXPmzEFycjKkUikAYNKkSbhw4QJmzpyJM2fO4PPPP8eqVaswbdo02189ERG5vSqFEoKBFokLN4rx4/5LOgm0l/NLTB9YIwhRarXArE2/qv7dlvkoBSXmB2LplwswbeUxDF+yD+fyinD/oh1YvP2cTrn/+yENvx25gq92X7BZPW3By5LCeXl5GD16NK5fv47AwEB07NgRmzZtwgMPPAAA+Pjjj+Hh4YHhw4ejvLwcSUlJ+Pzzz9Xv9/T0xLp16zB58mQkJCSgXr16GDNmDObPn68uEx0djfXr12PatGn49NNP0aRJE3z11VdISkqy0SUTEZE7unK7BCsOZGN0QnOEB/oCAM7fKMbQT3ajUYBUp/xfJ67jn8urE3rn/ynO9dxtpOtIxVOjGaVSqcTkH9PQvnEgkgfGoKzSeGuGhQ0wVrl4847698SPdgEAPtiUieSBMXrL5xWV10KtzGdRAPP1118b3e/r64vFixdj8eLFBss0a9YMf/31l9HjDBgwAEePHrWkakREREaN+vogsm7ewZ5zN/HHlL4AgI9S/kaFQomrBeLckWsFpergBQAqtFpg0u8mwxrj6XEvDNl+Jg8bTuZgw8kcJA+MQZXiXouMpIbhimbrjrmZLYJgeTdVXWNRAENEROSssu62OBy/Unhvo4En/phvDtb4fJqBSXG5uMVFOyDSVlRWhQc/3Y0HO4SbPE+V0vkTcq3BAIaIiEjL2bxi04XuKqtU4EBWPuKjg+Hr7am3zNvrxF1Qmi0w+vJv/jh2DQBw6rp4UlZBECAx0nRyMCvf7Ho7O65GTUREVAOv/XYcY745iLm/nzRd+C5rZ9Xt8nYKvt5jeoi1OYwFQvr8sP8S9l+w3xw4lmIAQ0REVANr06tbS1RzuKgYGtkEVCf1avs+9aLJcxWUVOLtdadw5bYZo6CMsHYVxBFf7seuv3WHfTsCAxgiIqJaptSTtzL3d/PncHlqqfH1AU1ZefgyMnOMrxmoUAp6R0sxgCEiInJh5rZyWDMK6VphmcXv0bZ4+3mj+wd/vBPt39yks/0rG3Vh1RQDGCIichnLD1zC8CX7cPtOhdFyX+/JqtHqzjVlaf5JbVMqBZy/cadOj3DiKCQiInIZr6+pTqT9bNs5zH24rcFyb687BaVSMJqnos1DAhh7nnt7ioMSQ0U1F1cEgNPXjXflOMJvR+r+Yo9sgSEiIpdzp7zKZJnjVwtNltGkOTGdPl4e5j1SF2w4I+o02pihf/VrR9p7zvRMw47GAIaIiFyOJS0r5vIw0e3jdbcF5te0K3jqi1Tk36lbU+/bkr7VtWsbu5CIiMgtWZqFYroFpnr/q78cAwAs3Jhpu5PXMlPhSdbNOyipUCCqgT8C/b1rpU7a2AJDRERkBpMBjKf4kVrXFj+0pddWn8BDn+3BzrOOG1LNAIaIiFyOqoejpKIKyw9cQq5cd9ixwsJuEFMBjLfWfnMWfKyrTN4ax/cgsQuJiIhcT5VSwPbMPKw+chV/HruGz/XMebL++HWLjulpMgfG/DaBojLTScZ1mSrHyJE9YQxgiIjI5aw5ehVrjl5Vv75aUFrjY3qY7EIy/3G+4kB2TatjN4JgOgVaNZzckdPZsAuJiIjIDDdM5LR4mzmMuq4zp2dNKahaYBwXwbjG3SYiInIwUzkyxtSV9YXMJbAFhoiIyDXU5GE++puDtqtIDQkwPc9LHcjhZQBDRERkC3VgbjebKKtUYJ2JBOdjd0dYOTKJlwEMERGRDbhI/IJf08xfB4ldSERERFQnFJRUWlCaSbxEREROrS6sD2QLH2/52+yybIEhIiIisgADGCIiIhtwkQYYizCJl4iIyMmZnr/W9Ugc2IfEAIaIiMgG2AJTuxjAEBER2cg76045ugq1ikm8RERETq60UoGv9mQ5uhpugwEMERGRDSiU7teHxBYYIiIicjpcjZqIiMjJuWMSryOzeBnAEBER2UCFQunoKrgVBjBEREQ2UFJR5egq1DoOoyYiInJyZZXu1wLDieyIiIjI6bAFhoiIiJwOh1ETERERWYABDBEREVmF88AQERGR02EXEhERETkdJvESERGR82ELDBEREZH5GMAQERGRVZjES0RERE6HSbxERETkdJwmiXfBggXo0aMHAgICEBoaisceewyZmZmiMgMGDIBEIhH9TJo0SVQmOzsbw4YNg7+/P0JDQzFjxgxUVYkXwdqxYwe6du0KqVSKmJgYLFu2zLorJCIiIrtwmrWQdu7cieTkZOzfvx8pKSmorKzE4MGDcefOHVG5CRMm4Pr16+qfhQsXqvcpFAoMGzYMFRUV2LdvH7777jssW7YMc+fOVZfJysrCsGHDMHDgQKSnp2Pq1KkYP348Nm3aVMPLJSIiIlfgZUnhjRs3il4vW7YMoaGhSEtLQ//+/dXb/f39ER4ervcYmzdvxqlTp7BlyxaEhYWhc+fOePvttzFr1izMmzcPPj4+WLp0KaKjo7Fo0SIAQJs2bbBnzx58/PHHSEpKsvQaiYiIyA6cNgemsLAQABAcHCzavnz5coSEhKB9+/aYPXs2SkpK1PtSU1PRoUMHhIWFqbclJSVBLpcjIyNDXSYxMVF0zKSkJKSmphqsS3l5OeRyueiHiIiI7MeROTAWtcBoUiqVmDp1Kvr06YP27durtz/77LNo1qwZIiMjcfz4ccyaNQuZmZlYvXo1ACAnJ0cUvABQv87JyTFaRi6Xo7S0FH5+fjr1WbBgAd566y1rL4eIiIgs5MgWGKsDmOTkZJw8eRJ79uwRbZ84caL69w4dOiAiIgKDBg3C+fPn0bJlS+trasLs2bMxffp09Wu5XI6oqCi7nY+IiIicJIlXZcqUKVi3bh22b9+OJk2aGC0bHx8PADh37hwAIDw8HLm5uaIyqteqvBlDZWQymd7WFwCQSqWQyWSiHyIiInJNFgUwgiBgypQpWLNmDbZt24bo6GiT70lPTwcAREREAAASEhJw4sQJ5OXlqcukpKRAJpOhbdu26jJbt24VHSclJQUJCQmWVJeIiIjsyGmSeJOTk/Hjjz9ixYoVCAgIQE5ODnJyclBaWgoAOH/+PN5++22kpaXh4sWL+OOPPzB69Gj0798fHTt2BAAMHjwYbdu2xahRo3Ds2DFs2rQJc+bMQXJyMqRSKQBg0qRJuHDhAmbOnIkzZ87g888/x6pVqzBt2jQbXz4RERFZy2kmsluyZAkKCwsxYMAAREREqH9WrlwJAPDx8cGWLVswePBgxMXF4ZVXXsHw4cPx559/qo/h6emJdevWwdPTEwkJCXjuuecwevRozJ8/X10mOjoa69evR0pKCjp16oRFixbhq6++4hBqIiKiOsSRE9lJBEEQHHZ2O5LL5QgMDERhYSHzYYiI3ETz19Y7ugpu5ffkPugUFWTTY5r7/OZaSEREROR0GMAQERGRVZwmiZeIiIhIReJs88AQERERsQWGiIiIyAIMYIiIiMgqCqXjBjIzgCEiIiKrKBw4EwsDGCIiIrKKki0wRERE5GyqGMAQERGRs2EODBERETkdtsAQERGR02EODBERkQ14OHBiNXfELiQiIiIb8HDk1LBuiF1IRERENsD4pXYltGjosHMzgCEiIpchYQRTqwL9vR12bgYwRETkMpgD4z4YwBARkcuQgBGMu2AAQ0RELoMtMO6DAQwREbkMVxiF9NOEXo6uglNgAENERK7D+eMXJLR03MgeZ8IAhoiIXIYjJ1ZzFv1ahTi6CjbBAIaIiFwGAxjTpF6u8eh3jasgIiICECbzdXQV6jxXyBMCGMAQERG5FS9PBjBERETkZNgCQ0RERA6V2CYUy8fHW/QeLxeZLMfL0RUgIiIi63w1pofF7/FwkQCGLTBEROQyBNS9UUjvPt7B0VUQ8WQXEhEREZnybHxTR1dBhEm8REREpPZMz7oVqBjCJF4iIqI6RnBgD1J5pcJxJ7eAqyTxMoAhIiKqoR7NG+BORZWjq2EWzSTeMJkU/3mmiwNrYz0GMERERCa0CKlndP+Q9hEoqXCOFhjtJN7mDf0dVJOaYQBDRERkwodPdcK8h9sa3C8IAgbEhtZijSxz/t0H1b97arTACAIgcdIlvBnAEBGRy7B1DszKib3w8dOd0LVpAzzfJ9roecckNLP6PDV5rzk0gxbNLqS6N+jcfAxgiIiIDIhv0RCPd2lispxSEODl6YH2jWVWnadhfalV79NnbJ/mRvdrJ/E666AkBjBEREQ1pGrJsLY7pqYxRN+YELPLag6jFgQg0M+7hmd3DC4lQEREVEPKu31X1s4EXNNWEKnXvfYIU0GUuAVGQJMGfujXKgRFZVW4daccl/NLzTpniA1bjazBAIaIiKiGVLk31ubgSGoYwTQKMD+Y0F4LSSKR4Idx8RAEATeKytHz3a1mHWfNP3tbVEdbYxcSERGRmQJ89X/vVyqrI5fi8tqbC2b+o+3Uv88cEofENmH4clQ3k+/THoWkIpFIECrzxfqX+uKdx9qbPE5UsGOHXzOAISIiMtOOVwfo3X43fjE4F8y3Y8WrRnt71iyRdkZSLEYnNFe/Dq7ng6/GdMfgduHo0jTI6Hu9TIxCahcZ6PDgxBwMYIiIyGUINhxHXV+q29rSsL4Un+mZuVaVA/PpiM4IkHrh8S6NRfsHas0R06tFQ/ye3Ef92pLk34c7RSJ5YIzB/Q91jMDHT3fCwic76t3fNuLeSClb3q/axgCGiIjcVvdmDQzuW/diX73bH+4UqbNNFQb0bhmCY28OxuvD2hg9b1Swv6j1Q7sFRna3q6pFI90ZgP28jT+6JRIJHu/SRGf24L9e6odPR3RGb40RS84bvjCJl4iI3FCLkHr4bXJvHLtSgOe/PaSzv1eLYDQ3sXyAJs2WDA8PCULqS7Fn1kDU89F9zMaFB2DWkDhcuFGs3qbd/vLHlL747cgVDIhthOFLUkX7qpTWhR1tI2VoG1nd+lJf6oXi8ip0a2o4gKvrLGqBWbBgAXr06IGAgACEhobiscceQ2ZmpqhMWVkZkpOT0bBhQ9SvXx/Dhw9Hbm6uqEx2djaGDRsGf39/hIaGYsaMGaiqEic+7dixA127doVUKkVMTAyWLVtm3RUSEZHbsOTR3qCej8FRQx4WJqUo9RyoSQN/NKjno7P927E9EOjnbbSuzUPq4ZXBsXrnaFGaGcB0bBKExkF+6BkdrLPvzxf74p8DWhrsZnIGFgUwO3fuRHJyMvbv34+UlBRUVlZi8ODBuHPnjrrMtGnT8Oeff+KXX37Bzp07ce3aNTzxxBPq/QqFAsOGDUNFRQX27duH7777DsuWLcPcuXPVZbKysjBs2DAMHDgQ6enpmDp1KsaPH49NmzbZ4JKJiMjVBfh64ZOnOxvc375xIADD87Z4elgawJgus2JCPL4Y1Q0RgX7V59Z4T1cDXVn6jmtuC4yPlwd2zRyIlRN76eyLDqmHmUPizJoBeNJ9LbFWI18HAF4a1MqsOtiTRV1IGzduFL1etmwZQkNDkZaWhv79+6OwsBBff/01VqxYgfvvvx8A8O2336JNmzbYv38/evXqhc2bN+PUqVPYsmULwsLC0LlzZ7z99tuYNWsW5s2bBx8fHyxduhTR0dFYtGgRAKBNmzbYs2cPPv74YyQlJdno0omIyFX9NKEX2jcOxNSV6QCAjk0CcfxKIZ7s1gQN6/tg8n0tAQBKpfh9A2MbYXvmDUzo18Lo8ZeM7Ip31p/G1YLqSd/MyYXt3VJ7ttx7b+rRPBg/jOuJ5g3F3VYKPcGKJa1DlgZiKo2DfNW/vzY0TrTvjyl90LFJkFXHtaUa5cAUFhYCAIKDq5un0tLSUFlZicTERHWZuLg4NG3aFKmpqejVqxdSU1PRoUMHhIWFqcskJSVh8uTJyMjIQJcuXZCamio6hqrM1KlTDdalvLwc5eXl6tdyubwml0ZETqKsUoFzecVoFymr8WRg2pRKAbdLKmy6To0tVSmU8PKsG2Mxjl0ugKeHRN2yoVJWqcCqw5cxoHUomjas/aG561/qi9IKBTo0CcTp60Xo2DhQNJFb75iGCJB6oai8CqN6NcNbj7TDzeJyhMp8jRwVGNohAkM7RKD5a+sBWDeaR/st/Vo10ikTGeSn/n3OsDZYtu8iZg6JtfhclooJDcCnIzojTM99iAmtb/fzm8PqAEapVGLq1Kno06cP2revnvAmJycHPj4+CAoKEpUNCwtDTk6Ouoxm8KLar9pnrIxcLkdpaSn8/PygbcGCBXjrrbesvRwiMuBOeRU+33EOQ9tHqB9Ohy7m4x9LU9EnpiF+HBdv88DBlNt3KiDz84anhwQvLDuEfedv4d3HO+DZ+KbIvlWCqwWlSGjZsMbneeG7Q9iReQO/Te6Nbgaa+K/cLoHUy9OimVBtIa+oDPct3IEHO0Rg0VOdavXcmsqrFPh+3yX8+6/TAICz/x4K77tB1Sdb/sYnW84CADw9TuH8uw/avT7aQUG7yHsBVeeoIJ3y/j5eSHvjAXh7StT/jk0FL+Ljy5BxTY5HOuuOTDKlfeNABPh6oXGQ7jNNJdDPG9tfHQCplwcig/ww3kTLkC092lk8FDzjrSRUKQT460lMdgSra5GcnIyTJ09iz549tqyP1WbPno3p06erX8vlckRFRTmwRkTO7aeD2fjf7gtoEVIfW07nYvH287j43jAAwD+WVo+K2HvuFp793wH8pNHHrlAKVjdbm+P0dTmGfrobCS0a4qeJvbDv/C0AwI/7L+HZ+Kbo/8F2AMAP43rq/UZriR2ZNwAA36de1Algluw4j91nb6jPr7o3528U41BWPv7RPcqu92HFgWyUVirw25ErtRbALD9wCYF+3nioY/XD+v2NZ7Bkx3lRmSqFgIKSclQqlOrgBdDfFVJX+HhZ34q1NrkPbt+psCjoUfH19kTanAd0VofWFm3BaCh7qqdnXhxHsupTmzJlCtatW4ft27ejSZN7y4yHh4ejoqICBQUFovK5ubkIDw9Xl9EelaR6baqMTCbT2/oCAFKpFDKZTPRDRNabvfoELty4gy2nc42WS71wCxtPVreefrb1LDrM24S/c4vwx7FruH/RDvydW2TTev18MFt93msF9xad89D6azbq64M1Ok9hSaX697JKBX46mI3rhaUoq1Rg+5k8vL/xjDp40TRo0U68tvoEVh66fO9YpZWY+esx7Dt/0+R5c+VlWLQ5E9cLzVtQr7Zczi/B62tOYsqKo+pt2sGLSo9/b0Hv97bVVtUcytvTw6rgRcXHy0NnbSIyj0UBjCAImDJlCtasWYNt27YhOjpatL9bt27w9vbG1q33FoLKzMxEdnY2EhISAAAJCQk4ceIE8vLy1GVSUlIgk8nQtm1bdRnNY6jKqI5BRI7z3b6LOts+2fI3AGBRyt8oqVBgwV+n8dJPR3Hhxh1M/Tnd4LEKSirwUcrfuHTrDkoqqlBRpcSWU7k4kn3brLr8a80J9e83iypwo6jcSGnz7Tl7E53mb1a/3pSRi9mrT+Dhz/bi9TUnMXaZ7rwh2jkQmtfw4aZMrDp8Bc/+74DJc4/77hA+23YOo40EYGmX8kWtG7WhQCOgM0bfcGKV/DsV5h1Dq7WmrFKBCd8fxooD2QbfU1JRhR2ZeahQKA2WIddiUXtQcnIyVqxYgd9//x0BAQHqnJXAwED4+fkhMDAQ48aNw/Tp0xEcHAyZTIYXX3wRCQkJ6NWruol58ODBaNu2LUaNGoWFCxciJycHc+bMQXJyMqTS6j7kSZMm4b///S9mzpyJF154Adu2bcOqVauwfv16G18+ERWXV+Gv49eR2DYMwXrmrND25h8ZJstoPn5KK/WvDbP77A11K8mSHecAAJWKe+9Udclo08y30QxYcuRl6PHvLQbrdPJqIepLvcyanOzjuwGZtpvF5fjtyBW9+xRKAV6e+hfJu5RfYvKc9+pZPQDhbF4xFEoBWTfvoGWjeqLr1p7YDKjOVfL19rRJt9WizZnYffYmfp7YC77enha9V2EkgFm+/xJeNDH8dvmBS3h/wxl8Py5enbPy88FspJzKRcqpXDwb31Tv+176KV3UWljLaVnkABa1wCxZsgSFhYUYMGAAIiIi1D8rV65Ul/n444/x0EMPYfjw4ejfvz/Cw8OxevVq9X5PT0+sW7cOnp6eSEhIwHPPPYfRo0dj/vz56jLR0dFYv349UlJS0KlTJyxatAhfffUVh1AT2cG/Vp/AzN+O4/lva9blokmVO2LIr2lXRF08lQpBFLzYWl5RGR76bA8GfLgDvxy+jNsGWgJqspKwsQe3tV7++SgSP9qJvu9vR5mBQBCobtlo9+YmPPTZvZxEeVklJnx/GH8eu2bxeT/bdg7plwuseq+xSda091zOL9G5rtfXnIS8rArT7g5/BgB5menPxVRXJ7kei1pgzBkm5uvri8WLF2Px4sUGyzRr1gx//fWX0eMMGDAAR48eNVqGiGpu/YnrAIDjVwrtcnzNvxvbM/Pw/oYzOJNjOi/m5NVC7D13E+P6Rtd4qHDaxXvdOTN+PY5OUUGihfQA4KOUv/GfrWfxxahuVp3D0J9HQRBQYiAwWn3kCrLzSzA1sTWKy6uw6W4ukcq649WfzdWCUny1+wKm3N8KFVW6XSQ7Mqu75E9fl0MQBEgkEnyx87y61ULf2j3artwuQWiAryihVbPVx9CEb9rMnWTt1DU5HvzPbjQO8sPe1+43WtaJ1xskO6pbKcVEVOts0dJu7gNmrJ41ZwxRtSb4S70wqlcza6oFoDqAmLz8iGjbscsFOuX+s7U6p+T/fkiz6jzao2xUD/zkFUdw+JL+nJ7pq44BqF6peOnO89igFcBoyrhW3bX0a5r+LiyVgR/uQELLEFgS8x2+mI8nl6aiXaRMtIChr4lFA/UxNtpI89/JxpP3grOaKqmwvuWMnFfdmAGJiJxCYal5iZy2dO7uKKZVhy7jf7suWPx+c3J2bMFQF9JfJ8RBSZVCqdOKUlhaaTR4Ae7NqGrqM7h4qwQ/HcxWz8Vijl8OVwdFGdfkoiRYX697+S8SjVD3i53nDXa32Wq4tGbLnanWn7m/637GEpuE5lSXsQWGyM1ZkuzY6a3NpgvZmKpLY+ZvxwEAiW1CLXr/96mXbF4nfXRyPww8cwd8uAN3yqtw8PV7s42b8xlY2o3iY2W3W7lGcCVKSta4oAUbzmDn3/rznIwFMD8euITg+j41alHT5/f0qzY9HjkHtsAQuTl7f1Ot6fdxqZd4FIzmN/88Gw2btgVzWx6u3C7F7ZJKfKnRmmTOZ1ClvWiPCZrBhyXGaQwRN3ZF+ubAAYzfhxtF5Xhj7UkUllZijYmgw9BRKhVKjP7mID7clGn0/eT6GMAQkY6TV+2T0GsNHy8PgwMIbDXviy1Y2nPygcYD+ODFfJPlN2Xk6k3gNcSSLiTN1pVDGgnPmlGEuYGuOUm8xy4X4HL+vdyXKoUSR7Nvo8qMOVwyrsmx6+8b+O/2c7hVXI7i8iq9I9g4jNr1sQuJyI0JgqB34i/NIaw1P0fN3l8dwNx7bclKvLXJ2ARupqgSiE35Pf2q2Q9mY9PjVymU8PSQmFzDSmlBHoq+9xhy+bZ4Xpy3/jyFH/ZfwuiEe11Lhg6j2TXW7Z0tGBhbs+UiyHmxBYbIjW0+pX/uDEvnNMnMLcLp6/pXgBcgYMupXPRbaN3U8l4ekhoFB7WlNtb6+XLXBcjNTKS+WaR/rpuiskp0e2cLxn132OQxrLnt1tyHH/ZX5ykZylfSrId2ILXdxJxD5LrYAkPkxq7eFg9hFQQBz397CBdu3LH4WEM/3W1w3/jvTT8sjbFHaHD6uhx/HruGF/pGI6R+zVeS1h1GbXtn84pxNq/YrLLf7M3Su33bmTwUllZi25k8vfs1WXMN5gQwH6eYbnEyt8WH3BcDGCJSu1pQanB0iSNpfgO3VQ+SKuA6cbUQP4yLr/HxHNlKZMk9Ka3QndHXUNUVFiYOA+blwNwstix3SfOI5t7mOtrTSDbELiQiUqurPTX2/DZ++OJts2YZN6U2upAM2WlBN8psjQUwp69KN1p20o9HMPf3k7heWGr2Qoy2mwfGJochF8YWGCI3VhvfUm3xIBK1wNTRCcqUgtbka7X4BF6bbv6aRZrVWn3kKv71YBuj5b9PvWTRXDr2aInS/MQZ2JAKW2CISM0dm90lEts8FJWCgCU7z6tfCwDWHbd8MUR72HP2psF9C/46Y9P2LXu0wFhzxLoa6JLtMIAhIruydQuMPZQYWenZXAqlgI9T/hZtm7KibixI+9zXBwzuu3XHtnPp2Lsrjcm9pMIAhsiNaX9HNTUviCNculUi6pawpIrLD5jX9dHz31ssrZYOhVKAl4fz/UkVBNsGiOauRm0RUdec7Q9Pzok5MESkVvfCl+o5Qqz91v36mpMmy0gA3NEzMsdSb/x+EqU2aMmpbbaOB3TWhLLSjaJyvL7mBJo19BdtZ/xCKgxgiNyYdovLnnOGcyUc6cf92erfHbEitjmOZheIXpdXWj4E2VFs2S1zOkf/hIaWqlAosfxA9ef+4v0xFr+/DjYmko05X3snEdnNzF+PO7oKJh2/UnfWaTJmY0aOo6tgFkEQUFxWZbqgmRZutO8ii7U5uovqNgYwRGRXVwtKTRcih1EoBYNLStQVomHUDqsF1TUMYIiI3Jgt8n/szaqZeO1SE6pLmANDRG7NGR7gtnC1oBSeehJDjl0uqP3KENkAAxgiN8ZER/fR5z3rVgOvCzRbXf67zfRCkOQe2IVEREROY7sF6z6Ra2MAQ0REdZo1w7zZuuj6GMAQuTH+jSciZ8UAhoiI6jQuzEj6MIAhIqI6zbqZghn0uDoGMETujIkCROSkGMAQEVGdZt3qAZyz19UxgCEiIiKnwwCGiIjqtIxrlq9wzTUfXR8DGCI3xgwYcgY7/7Z88jolAxiXxwCGiIiInA4DGCIicjnWDb0mZ8IAhsiNcRQ1uSrmwLg+BjBERORyGMC4PgYwRERE5HQYwBC5Ma4xQ66KOTCujwEMERG5HHYhuT4GMEREROR0GMAQERGR02EAQ+TGOIyaXBW7kFwfAxgiInI5TOJ1fQxgiIiIyOkwgCFyY+xBIlfFLiTXxwCGiIhcDuMX12dxALNr1y48/PDDiIyMhEQiwdq1a0X7n3/+eUgkEtHPkCFDRGXy8/MxcuRIyGQyBAUFYdy4cSguLhaVOX78OPr16wdfX19ERUVh4cKFll8dERERuSSLA5g7d+6gU6dOWLx4scEyQ4YMwfXr19U/P/30k2j/yJEjkZGRgZSUFKxbtw67du3CxIkT1fvlcjkGDx6MZs2aIS0tDR988AHmzZuHL7/80tLqEpERHIVErkpgH5LL87L0DUOHDsXQoUONlpFKpQgPD9e77/Tp09i4cSMOHTqE7t27AwA+++wzPPjgg/jwww8RGRmJ5cuXo6KiAt988w18fHzQrl07pKen46OPPhIFOkRERPowfHF9dsmB2bFjB0JDQxEbG4vJkyfj1q1b6n2pqakICgpSBy8AkJiYCA8PDxw4cEBdpn///vDx8VGXSUpKQmZmJm7fvq33nOXl5ZDL5aIfIiJyT2yBcX02D2CGDBmC77//Hlu3bsX777+PnTt3YujQoVAoFACAnJwchIaGit7j5eWF4OBg5OTkqMuEhYWJyqheq8poW7BgAQIDA9U/UVFRtr40IiIiqiMs7kIyZcSIEerfO3TogI4dO6Jly5bYsWMHBg0aZOvTqc2ePRvTp09Xv5bL5QxiiEzgatTkqtgA4/rsPoy6RYsWCAkJwblz5wAA4eHhyMvLE5WpqqpCfn6+Om8mPDwcubm5ojKq14Zya6RSKWQymeiHiIjcE+MX12f3AObKlSu4desWIiIiAAAJCQkoKChAWlqausy2bdugVCoRHx+vLrNr1y5UVlaqy6SkpCA2NhYNGjSwd5WJiIiojrM4gCkuLkZ6ejrS09MBAFlZWUhPT0d2djaKi4sxY8YM7N+/HxcvXsTWrVvx6KOPIiYmBklJSQCANm3aYMiQIZgwYQIOHjyIvXv3YsqUKRgxYgQiIyMBAM8++yx8fHwwbtw4ZGRkYOXKlfj0009FXUREZAPsQSIXxS4k12dxAHP48GF06dIFXbp0AQBMnz4dXbp0wdy5c+Hp6Ynjx4/jkUceQevWrTFu3Dh069YNu3fvhlQqVR9j+fLliIuLw6BBg/Dggw+ib9++ojleAgMDsXnzZmRlZaFbt2545ZVXMHfuXA6hJiIis3AUkuuzOIl3wIABRv9hbNq0yeQxgoODsWLFCqNlOnbsiN27d1taPSIiIubAuAGuhUTkxtiDRK4qXObr6CqQnTGAISIil9M8pJ6jq0B2xgCGiIiInA4DGCIiInI6DGCI3JiEy1ETkZNiAENERC7l/rhQ04XI6TGAISIil5LQoqGjq0C1gAEMkRtjBxK5IvaMugcGMEREROR0GMAQEZFL4SoC7oEBDJEb4995InJWDGCI3BgXvCNXxBwY98AAhsiNMXwhImfFAIaIiIicDgMYInfGJhgiclIMYIjcmMAIhoicFAMYIiIicjoMYIjcmKcH/wSQ7X0+squjq0BugH+9iNxYPR9PR1eBXNCpa3JHV4HcAAMYIjfmThkwP0/s5egquI3IID9HV4HcAAMYInILAb5ejq6CzbSLlDm6CkY90bWxo6tAboABDBGRA/h4Wv/n16OOTzXr682uSbI/BjBEbsydVhKQoG4/9C3Ru2VDR1eByOEYwBCRW6jjjRYWeaZnU0dXwSbGJDRzdBXIiTGAIXJjnMjOOXl6SODt6diI7Pi8wTU+hsTCqLKBv7dZ5e5r3cia6pCTYQBDRG6hNltgRsbbt4VEInF8l5jM17xgwpb+N7q7WeVahQXYuSZUFzCAISK99r12v6Or4LSaBvsb3W+TId0u0iX2QNsws8syMCFNDGCI3JixJF5Xm8ujNlssTHXM9WpRsyRcicTR7S+2M/2B1maX9TDjoh/rHFmD2pAzYQBDRG6hrFLh6CqQHrEWtKqYkzPzyYguNakOOREGMERuzFBLwf1xoSbfa8mDpy64XVLh6CrYjASuM6rKw0OCOcPamFfWRa6ZbIMBDBHpiAzyNVkmVCZV/x4RaLq8o7la/oTrdCKZPzFfXZ/Aj2oXAxgiqrG6vs7Qr5MS0DjIDztnDHCpJQVchblxCeMX0sQAhsiNCTWYirexRpJvs4b1bFEdgxb9oxNSZ1s/Kio2vLr1pVnDemZ1jzmaqTleJBLXepizBYaswQCGiKzySOdITLqvJb55vnpuDntOrCaRABGB1o+K0nzw1cYjsKZLNBx6PdFkGXOuo1lD48O56wqzW2DsWw1yMgxgiEiHOQ9gCSR4bWgc7o+rnsejJosTamrRSLc1R1nDgMBTI/vT0tlfHcHDRtmqW6bfZ5Pj2IuqBdDcz4QtMKSJAQwRWUV7GQIfL/v9ObGkq6tNhExnm+ZzzxkegaYe1BJIzHroe3t6oFVofavqMC2xNY7NrflyAeYwN15j/EKaGMAQkQ5rJrFb8EQHO9SkmiUNMBte7qezzbMWnnwxZgYK5qzT4yEB/q9/C6Nl7H1FPZo3QKCJtYe+HdujRudQBWHmjqhyhtYzqj0MYIjcmKpho2vTINH2cX2jzXiz+OWQ9hHo0DjQJvWSaY0UqkmyMaDVomGnZ6C5hzVnzhMPSXX33Jbp/fGsoXWV7Nxq0TEqyGSZgbGGE6ItOS/ndyFrMIAhIvj7iAMGX29PAMC6F/viqe5NsPWV+/Dt8z0wvGsTdRl9IUV9qW2GKG+a1h8Lh3e8d64a5sBo5pTYK49C87Cnr8vVv2vnBqnurTEekuouopjQALMDo6Htw80saZ6afpbH3hws+gyN0feZtAix78g2cn4MYIjcmHYei7b2jQOx8MlOaNmoPgbGhULzWWxuUPHS/TFY92Jfs+skQfWIo6d6RGnU03a8bZRsDIhbijTvx4mrhfhqdHe0CKmHXyYl4PsXelp0XHNaJMb3FXcxRZlYQLK2yXy90STYvK5IfTHl+H7Gu9CIGMAQkdnN/ZqJuvqCH33HCfT3QXsjXUtx4aZnyK1pC4wmHz3Dvec+1BYbp+rmzlhCs2WlSQM/JLYNw7ZXB6BTVBA6NrGsa82cVqJRCc1ErxNquECkPZi6DlXXYM/oYNH2Z+Ob4h/dm+h7i9rzvZujX6uQmlWQnBoDGCICIB5qbMhLg1qpfzc3qBiplcOhfZ7l4+NNHsNUS5ElNFtg3nuiAz56qhNe6BuNuHDd0UuW0HxWVykErX33dirNuHGmhlFLJLopMANiG+G7F3pi8bNdRdsrFabPN96cnCcraAYw0x9ojfS5D4j2q+6L9kSI7z7ewWRLWbtIGaZZsJI1uR4GMERuTPNZ+tOEXmgc5KeemE6f0IB7ax6ZG1KoWiaWje2B6Q+01pkJ15zASfuZ37N5sP6CZvDWaEUa0bMpnuhq+Jv+zhkDdLa9+XBbk+coqxKvfK15iYbiF3NHMRkikUhwX+tGCNdal+rJbsZbMgBgrAUBTD0fTyS2CTOrrOZ1vzSoFYL8fUT7azr5IXN/3RsDGCICUN2Mv/e1+9UT05kSbeHyAQNiQ/HSoFY6Dx1jrQ2JbcLg6+2BYR0iRNt9fUwnwhpiSQ5Ms4b1RAHL092jEFzPx8g7qoXLxEGEZpCmrwVm09T+eMVAa4KhQNHcbj9zRpQ1tmDYfPqbg/G/0d3MKmtq2LNXDfKRujVrgI5NgtC9WQOrj0HOjQEMkRuzJrdk09T+WDEhHk31TFM/wYrES2NztPxvdDccfzMJDcwIGsylLwfGmLF97gUAAgTRQ9nQA/qtR9qJXnuIupB0y8eGBxgMSEorFDrbjF2Bv1Zw5+vtibkPmW41Mpe3p4fZ87Hoa117pue95GxrEqrT5iRi87T+aNGoPjw9JPh1cm/1PmdZOoFsw+J/Pbt27cLDDz+MyMhISCQSrF27VrRfEATMnTsXERER8PPzQ2JiIs6ePSsqk5+fj5EjR0ImkyEoKAjjxo1DcXGxqMzx48fRr18/+Pr6IioqCgsXLrT86ojILJZMEBYbHoDeLfUnTw6MC8X+2YPw9qPt9O4HdFsUtB9y2tP+65vhtyZdB13t9I09UmOtplCtFhjx7bUsaiwur9LZ5uvjCanXvUCls8acLXHhARjbpzlmD41Tb3u+d3OEyaSiY6hyk+yV/wIAHRoHonNUkKgFbcET94ZWawaTj3dpbPRYqqCwYX0pWofpT/yemthK73ZyTRYP9L9z5w46deqEF154AU888YTO/oULF+I///kPvvvuO0RHR+ONN95AUlISTp06BV/f6v+pR44cievXryMlJQWVlZUYO3YsJk6ciBUrVgAA5HI5Bg8ejMTERCxduhQnTpzACy+8gKCgIEycOLGGl0xE9hQe6Gs0IAoNED9Itb+Fa89Jo4+hwy99rrpr49MRnfHyz+l6y/RuGVI9xFnPmkuaHu4UabIemuY/1g4SCfBcr2Y6+zRbmQy3eum/qDsaAcwXo7pBEATIfMUz5GrOgiyRSPDmw1otQB4SdGvWAH+dyFFv6xQVhDceamvWvDTW8vSQYG1yH4P7NbuQxvZpjjVHrxoclfZYZ8MBzrfP98DBi/l4pJPxIIhci8UBzNChQzF06FC9+wRBwCeffII5c+bg0UcfBQB8//33CAsLw9q1azFixAicPn0aGzduxKFDh9C9e3Wy4GeffYYHH3wQH374ISIjI7F8+XJUVFTgm2++gY+PD9q1a4f09HR89NFHDGCIbMiW86toUk2X30DPVPQzkmKx/EA2gOqFGz09JNg1YyD6f7AdAFBPav0Ddcjdydwe7dwYFVVKzPj1uN5yiW315/lMGRiD9MsFWPJcV4MTuXU0MCQ8NMAXS57TnxuirwtJOwgzFJS9mhSLJz7fh7F9miOpnXiyus3T+iP7Vone9Z+0zR7aBocv3kZeUTkAIKqBv8Hg5a+X7g0p/2FcT4z6+iAA45Pb+Xh6oEKhNFkPTZrBa8cmQdj32v0IqS818g79BsaFYmCc4VmByTXZNAcmKysLOTk5SEy8txR8YGAg4uPjkZqaCgBITU1FUFCQOngBgMTERHh4eODAgQPqMv3794ePz71+76SkJGRmZuL27dt6z11eXg65XC76cXZKrc7yCzeKseboFZ3t9iIvq6yV89RVZZUKZN284+hqqB2+mI/rhaV2ObatR3NEBftj/+xB2PfaIJ19Qf4+OPP2EHz8dCes+r8EABDl09QzpwUGwJKRXdEuUoaBsdXB0hCth3vvmOpuLktmlH01KRY/jo9HgK+3wVak5iH1sO7Fvtg/exAC/YyvFaSur8ahVO/57oWeaODvjc+e6aK+Jn26Nm2AU/OTdFpVAKB1WIDBYExbVLA/DvxrEH6a0AtvP9YeCS3F88a8cTdP5qOnOqFt5L2AqF+rRtjwcj8MigtVf176zHmoDZ7v3RwrJ/Yyqz4AdJaeiAzys+uioORabDPv9105OdXNk2Fh4v+hwsLC1PtycnIQGiqOlL28vBAcHCwqEx0drXMM1b4GDXT7sBcsWIC33nrLNhdSA4cv5qNpQ3/RcFNTisoqEaDVJPzD/kv4cFMmfhwXjw53J8G6f9FOANULnz1mor/YEreKy7HldC4e6hiJenf/2C/bm4V5f57C24+2w6iE5gbfW1haie/2XcQjnSLRvI5P/Z1XVIbnvzmE4vIqjO3TXJScqSmnsAyzVx/H9swbAIAV4+PVD8PaJAj3EkaPZt/Gk0urvwRcfG+YTY5fpVDiwo1i0wWtpD2cV5Ovtyce7yIe3vvq4Nb4du9FvG5krSAvDwmqlAL6xIRgaIcIDO0QgUqFEgez8tFNK7elcZAfUmffr9PdYo248ACcySlS11k1Md8Xo7ph2sp0zEiKNfp+iUSCL0Z1w53yKvV96dWiIY688YD6MzbWimJOt5o5JBIJElo21AlegOrRSiN6RKn/BmhqEyHD18/rX7jx34+3R+r5W3imZ1Ozk3I3Te2PrJt3dCaw07Zxaj8M+WQ3AKC+r00fV+QCXOZfxOzZszF9+nT1a7lcjqioKCPvsL39F25hxJf74SEBtky/D9Eh9UwmR27OyMHEH9KQPLAlqpQCjl0uwA/j4vHG2pMAgGmr0rFl+n2i9xzJvl3jAKasUoH3NpxBYpswvLP+FM7kFOH09SLMu5soN+/PUwCAN37PMBrAzFl7En8eu4alO8/j1PwhSLuUj6PZBRjXN7rOrRz70ea/ceruGjVv/XkKI3o0xekcOUrKFeirMaPnnLUn1cELAPyadsXqAKakogrplwvQs3mwRUNGz+YW4ekv9yN5YAzG9Y3GoYv5Fp33XF4R/r3+NF5ObC1K8NT00s9H1TkRdeGjmnJ/KyQPjDH672bHjAE4mJWPRzTyU7w9PdDHwOcTEWj5qtr6/D6lD3IKy3QmXGsTIcPGqf3NOoZ29w8gTp6OCvbHH1P6oIG/7UZcWUpf8GLKyPhmGBmvm/djTGx4AGLNmIE5LlyGzHeGADBvviByLzZtqwsPr/4fNDc3V7Q9NzdXvS88PBx5eXmi/VVVVcjPzxeV0XcMzXNok0qlkMlkoh97WnXoMsZ/dwjlGhNWbT9TfV1Kobq15Me7/fz6qKbQfvOPDADA4u3n8cXOC9h/IR+7/r738FT9L1tWee88mvNICIKA9zeewYYT1y2q/5e7LmDZvot47usDOJNTBADYqXFecx24cAsAUHJ3qOfwJal4Z/1prLewPionrhRi1983cC6vGA9+uhsbT+aYfpOZSivFw1EVgoAnPt+H574+gLyiMvV2zd8BoNJEl51CKSD7VonefRO+P4xn/3cAn207Z1FdX19zEvl3KvD2uupAUqLRwZB6/haqTOQajF12CNszb+CxxXsNltFM6KwrTAW9TRr444muTWo0f4g1pF6eOsGLPXRsElTn1jRyNKmXeMQVkYpN/wpER0cjPDwcW7duVW+Ty+U4cOAAEhKq+04TEhJQUFCAtLQ0dZlt27ZBqVQiPj5eXWbXrl2orLyXg5GSkoLY2Fi93UeOMPO349hyOg+xczaqt2k/IP+z9az22wBUdw/1+PdWZOYUmb0yrmZwofk83XI6D0t2nMfk5UcsqD1wOV/3gSszsz8fqB7auXTneXVCoLZzeaa7Jo5k38b+uwGQysP/3YPR3xzEP5buw6nrckz6Mc3Au8UUSgFXbusPIsxxs6jCyLGVyJOX4eTVQr37X/rpKPp/sB1rjl7R2bf3XPX1fbr1LMZ/dxhpl25j8fZz+C1Nt6yKIAg4qNXiovnP5Jn/7ceHm/8W7a9SKFFRdS+ouZxvn1wZIqK6wuIApri4GOnp6UhPTwdQnbibnp6O7OxsSCQSTJ06Fe+88w7++OMPnDhxAqNHj0ZkZCQee+wxAECbNm0wZMgQTJgwAQcPHsTevXsxZcoUjBgxApGR1c3Czz77LHx8fDBu3DhkZGRg5cqV+PTTT0VdRHVRidaEU156mjwLSyvxxtqTuFlcjlm/6R8hoZ3EtuHEdUxZcS9AETRaYLRbCzRVVCnx1e4LOJtbpLOvpl0G8//MwHsbzhjcbyrPWKmsbv0Y8eV+/JB6UTRUFABul1iWQDzpxzT0fX+7qMWmsKQSTy7Zhx9SLwIwPmnbjeJyzPsjA3/ruVeVCgE9392Khz7bo/deqlqbluw4b7SOW07nYviSffhgUyZe+eUYLueXiD5LFVWLmCbtQPe7fRfVvwuCgPs+2IEe/96CSgtHgaiwcZ6InI3FAczhw4fRpUsXdOlSnTk/ffp0dOnSBXPnzgUAzJw5Ey+++CImTpyIHj16oLi4GBs3blTPAQMAy5cvR1xcHAYNGoQHH3wQffv2xZdffqneHxgYiM2bNyMrKwvdunXDK6+8grlz59b5IdTaLTD6+mwHf7xT/XtZpQJXC3S/Ke8+e1P0evLyI6IF2cyZPfXLXefRes4GvLP+NB74eBeA6mThd9adwrHLBaYPYML+CyZyMoxUcvWRK+g8f7P69Ru/ZxgM5syVcqq6i/F/uy+otyWvOILDl27jjd8z9L5H89MZ881BLNt3EYPv3itNCo1o7KgN7p1Kv4Xb8e/1p3W2ay8ECBgfclulFHC1oBSFpZW4cpstL0TkHizO2BowYIDeb40qEokE8+fPx/z58w2WCQ4OVk9aZ0jHjh2xe/duS6vnUNp5CfpaYHLl97pc9H3TBqrzU4yex4xh1O/+pds6snBjJn7Yfwlf7cnC093tm+BsrIbTVx3T2bbu+HX899man1ezBWLPuZtGSlp3TGOLz1kzLf9Xe7Iwx8Q071NWHLG4ZUUiMV6fa1qBc11LuCYiMoUD7u3I2CJ15tL3DPo17YrRINLQvkyN7g9Lnld/HLumbuEwl74F62pCXlaJMzmm5/bRzAOxlnbVNY/p6WH4fxlbXbGgdaR1x69jU4b4/mt+fPputeb+wtLq7rgf91/CM1/ux6aMHPR+b5uNaktE5BguM4y6LtB+kBhbpK6mFEoBXlqtAZUKJYrLquBvaCZTjfrpq5qh2r7001EAQJem1WuajO/XwmQAZKiRyFjgZUzf97ZBXlaF3yb31pnvQ5MlLRWGalJQKk7oPZB1r7vM20hQei6vGAez8k3ObWELlrSYfL/vIr5LvYSbxdWtf6laidNERM6ILTA2kCuvTqTVfmjbYt4CS47w8Gd70OXtFJzP0z97rPY3e0sdzS7AO+tPI/+O4RE7KrZsgVEoBcjLqpN8VUPVDanUkz9iKWMjeEwN333qi1SLz1dWqbvasCmqfxeZOUU4fqXAaNlFKX+rgxdTxyMichYMYGxg0ebMu7+JH572nHhJ32NalVOz7vg1k+8/m2v9DKxd307BJQPznqhpVHDDietIu1S9BIQ1cY1mq42phgdT86PUlGarV2FpJbadMdy1pjBzyYcj2eLlMSy5R0mf7FLP0FsTxw0MESciqqvYhWQDqhwJ7QePo/IiDSX5atbv8CX9a0rZiqoF5mxukXqOGmunwNe8Gn23VHPOmQoDLTBb9OTwWNOd5a2RA/PcVwdwwsCDf9Why3jj95MWH98SY789aHCfxFQWr5YbBubzISKqq9gCYwOGHhM2zmM1+9i2SGStKVUMpT2s15pbIrpWiUS0mOX2zDwkfnRvaLqhrqvx3x/WPa4VdVl95ApKKqq7swwFL2WVCsz87TjKzfwcisqq8HHK3zhvwbpEReVVouUOVH5NuwzA/NYfIiJnxRYYGxAEIPtWCbaayM+w6TmNPH6rlPofnLdLTOeu2Iotc2A0r/U/W8/i54PZePH+GPyw/xK8tEYFWdKqYk0VVx+9Ch8vD7w3vKPBMh3mbbLomG/9kYFrhWX47/ZzOP/ug5ZXSsPi7ecxIymuRscgInIGbIGxkZFf79fZZotnuDWtO4a+fJ+/oT+51x7U9ZNob7f8pmi/Ja+oHG/8noG/c4vVizOqGGt4SNeahM7YdP7GbDYxpNzSROJrhdVJ4AqlgCqFUmdWYiIi0sUWGBsQoH/kiuoxVlqhgNTLw6p5YcxZU4juUbX8fLMnS2dfttb6T/PvLpRoKXumNr371xl8s1e37pYY+qlzTQBJRGQNtsDYkSAIuFlcjjZzN+LpL2s+UsRcOYWG10eqLapAwpKHfep5/fOTWNJooyprbXBiDnsmZ9c0eAGA09dNT/hHROTsGMDYgLFukU0Z1YsLHrp42+qF9vSf0/C+bVbm4giCgEMX81FaYfm8JLrHqv6v9oRrxmKRlYey9W7fnmn+9QiCIEryJSIi18QuJBtYd/y63u3aQcakH9Jsds7yKgX8fAzMuGulY1cK8Y+lqegT07DGx7KmBcaQfy4/YrqQ+rzA7NUnbHBWYyR1YqQXEZE7YwtMLbLlKKXO81NwOd/EZHJW2nuu5lPNH7tSoNMyJQiCXYeWA9UjllYevmzXc9wsLkfrORvseg4iIjKOAYwd1XTqflN+2H/JrseviZNX5diUkSvKFzE1N8nadNMzCJvC3iMiIvfAAMaO7N7aIAiQ1OFVbP48Jg5IlIL9gzp7H56IiOoGBjB25G7PUu0uI6VWgGXLye0MqY1zEBGR4zGAsaPrBaXYc/amXc9hzUrG9rL+hDiZWTuWEATzW6XaRsisqgMDGCIi98AAxo7uVCiw4WSOXc9hz/lOLDVlxVHRa6UgiHJgLAkugvy9raoDc2CIiNwDAxiyGwHiYdQKCwIYR63kTUREzoEBDNmNzjBqTp1CREQ2wgCG7EbQaoJRWjAPjC3moiEiItfFAMaJfbfP/HlgCksq7VgT/bRzXphgS0REtsKlBJxYhQVrK5U6YLSSdrjy6dazKLHBOktEREQMYNxEYWntt8AolALe33BG/fr71Lo7czARETkXBjBu4j/bztb6OXfbeQ4cIiJyX8yBcRMHLuQ7ugpEREQ2wwDGTdwsLnd0FYiIiGyGAQwRERE5HQYwRERE5HQYwBAREZHTYQBjBQVXDCQiInIoBjBWmPdHhqOrQERE5NYYwFjhh/2ckI2IiMiRGMAQERGR02EAQ0RERE6HAYyF1h+/7ugqEBERuT0GMBaa9dtxR1eBiIjI7TGAsZBS4BBqIiIiR2MAYyHGL0RERI7HAMZCAhjBEBERORoDGAtxEl4iIiLHYwBjKQYwREREDscAxkLsQiIiInI8BjAWqlQwgCEiInI0BjBERETkdGwewMybNw8SiUT0ExcXp95fVlaG5ORkNGzYEPXr18fw4cORm5srOkZ2djaGDRsGf39/hIaGYsaMGaiqqrJ1VYmIiMhJednjoO3atcOWLVvuncTr3mmmTZuG9evX45dffkFgYCCmTJmCJ554Anv37gUAKBQKDBs2DOHh4di3bx+uX7+O0aNHw9vbG++++649qktEREROxi4BjJeXF8LDw3W2FxYW4uuvv8aKFStw//33AwC+/fZbtGnTBvv370evXr2wefNmnDp1Clu2bEFYWBg6d+6Mt99+G7NmzcK8efPg4+NjjyoTERGRE7FLDszZs2cRGRmJFi1aYOTIkcjOzgYApKWlobKyEomJieqycXFxaNq0KVJTUwEAqamp6NChA8LCwtRlkpKSIJfLkZGRYfCc5eXlkMvloh8iIiJyTTYPYOLj47Fs2TJs3LgRS5YsQVZWFvr164eioiLk5OTAx8cHQUFBoveEhYUhJycHAJCTkyMKXlT7VfsMWbBgAQIDA9U/UVFRtr0wIiIiqjNs3oU0dOhQ9e8dO3ZEfHw8mjVrhlWrVsHPz8/Wp1ObPXs2pk+frn4tl8sZxBAREbkouw+jDgoKQuvWrXHu3DmEh4ejoqICBQUFojK5ubnqnJnw8HCdUUmq1/ryalSkUilkMpnoh4iIiFyT3QOY4uJinD9/HhEREejWrRu8vb2xdetW9f7MzExkZ2cjISEBAJCQkIATJ04gLy9PXSYlJQUymQxt27a1d3WJiIjICdi8C+nVV1/Fww8/jGbNmuHatWt488034enpiWeeeQaBgYEYN24cpk+fjuDgYMhkMrz44otISEhAr169AACDBw9G27ZtMWrUKCxcuBA5OTmYM2cOkpOTIZVKbV1dIiIickI2D2CuXLmCZ555Brdu3UKjRo3Qt29f7N+/H40aNQIAfPzxx/Dw8MDw4cNRXl6OpKQkfP755+r3e3p6Yt26dZg8eTISEhJQr149jBkzBvPnz7d1VYmIiMhJSQRBcMnFfeRyOQIDA1FYWGjTfJjmr6232bGI6pKL7w1zdBWIiMx+fnMtJCIiInI6DGAsFBce4OgqEBERuT0GMBbq0rSBo6tARETk9hjAWEgicXQNiIiIiAGMhRi/EBEROR4DGAuxBYZc0XO9mjq6CkREFmEAYyEJ22DIBb3xEGe5JiLnwgDGQmyBIVck9fJ0dBWIiCzCAMZCjF+IiIgcjwGMhSRsgiEiInI4BjBERETkdBjAEBERkdNhAENEREROhwEMEREROR0GMEREROR0GMAQERGR02EAQ+Sm4sIDHF0FIiKrMYCxkCAIjq4CERGR22MAQ+SmGIsTkTNjAEPkpgQwgiEi58UAhlwGV3mwDFtgiMiZMYCxk1ah9R1dBbfD+IWIyH0wgLETDzYHWO2b57s7ugp1Vn2pl82OxX+iROTMGMBYyNxWd2MPh/aNZTapi6uytmvDHXpEGtTzttmx2IVERM6MAYydeHvy1lpLaW0A4wYPZE8bNps08Pex2bGIiGobn7J24unh3O3zDfxt903fUu0ijbdQvfFQ21qqiWPcHxeKVx5orXefLbsmGwVIbXYsIqLaxgDGQuZ+y/cyEsA4Q0uBIwOwyCA/tGhUz+D+cX2ja7E2te+b53vgxUGt9O57OVH/dqs4d4xNRG6OAYydeHk699PhtaFtHHr+DS/3s/kxA/0c16pkK492bmyzYzn3v1AicncMYOzE2XNgujVr4NDzS708bX7Mkooqmx/TmUk4DImInJhzP2XrMGNdMM7QhWSsC6yuigj0Nbp/zjDnzp3p1CTQ0VUgIqozGMDYiZeHc99aDzMCmKR2YfhhXM9aqI15jCW4JrRoiISWDWuxNrbXKSrIpsdzvhCViOge537KOkDr8ACzyjljC4Ymc+rfq0VD9GvVqBZqU3P+Pp5O/8C29eSI7EEiImfGAMZCz/SIQtNgf5PlPJ08idcZZxIWjPTNlVQorD7uon90MrusPeNWW38kzvcJExHdwwDGQl6eHpjYv4Wjq2F3lrQgmRPQOdr1wlKr32tsSLc2e6Y32b4FhiEMETkvBjBWMPZN31VotiCN6BGlt4yqhD2GPNtaeZXS6hYMAUBUsB8AIEzmuMnfbBlu/J7chy0wROTUGMBYoXEDP6P7vT0lRqd8FwAE2HBRPgBoE2Hb9ZU062/oUlRhnDPMOqysQdApCMCfU/pi2dge2PfaIBvWyjKqxOr/u6/mLYAdGnNEExE5NwYwVhgYG4pZQ+JwX2vdBNaMt5JwYl6SyQfmTxN72at6Znu4UyS+HdtD7z5xUGI8QNEMcDZOdVxrjLE7rlACNWnDCPL3wYDYUIcGa6r7PCspzjbHqvtxJxGRQQxgrCCRSDB5QEvMflD3QVJP6gVfb0+TAUzD+o5fSE8C/YsDSiTifAtDLTD6Nkvq6FOxZt1+9usyfLyL+TPrqj4TDw8J3n60XY3OK5FI6uxnRURkDgYwNRAXbrjbRmFiSWVrV1zW5/3hHcx+FPXWmgtFXzUy3koSjabx1ZgV98luTXTKG3oQDoxtZLN1i3o2DzZZxtg9UAiC9Tkwdkx5siQxV/MzGZXQ3Kz3TOhn+P4zh5eInBkDGDup7rLQTxAEq1oEDHVfhMkMz0Ar9RJ/xCsm3Ou6MlQDfx8veHl6YEZSLP45oCUig+4d/+1H26t/V41iMfQgHN27uU1Wjg6u54O4CNPz7xi7o8oaRIyWvNPSj1Ww4OjWjEJ6XWP2YV9vD7z1SDv1sHDGL0TkzGybSUpqCqXhCCapXbhV3+oNPXCMrbv0RNcmeGlQDFYduqI3CNAOpDSDpOSBMQCAr3ZfUG/z8zG+RpElz1gvDwmqbNkUZYT2afx9PFFSoUCP5g1w6OJto++166AzrWO/P7yDwaLaw577xDTE3nO3zD6Vt4cHxvRurnE8s99KRFTnsAXGThRGHnrJA2OsSgY19MAxdayIQD+8nNgKSe3CTZ5DX9eXqflCDO418eAfndAcI+ObmqyTn7d5s+gO6xBhcJ9SEETHWP3P3jjwr0Ho0DjI5HHtOWxe88hn3h6Cp3sYvh/aH/OXo7rjt8kJJs/x+oPVK4t/YMGEfEREdR0DGDtpEWJ48jMfLw9EBPrikU6Rou39WoVgeNcmeOex9nrfZyjXxMtDYjjR1sST3xaPZnMnRPPR6s7y9pLgX3cfrsbI/LzNOv6rSbEG92kHZt6eHgiT+ZrVCqF9j4L8DdcnUKOuxv4NqGgme/t6m2jd0vr860m90K1ZMJo3ND6R4IT+LXDm7SEY0l4cwDKJl4icGQMYO3llcGs837u5wW/IEokE/3mmi872RU91wnO9muk/qATqh1WIxigma4f2CoIAb41FJxv4e+OLUd10yhnrDtNTRYv2mRNA9GsVYta5jQUACqUgCrRq8ujW9956Pp5oGyETLW75ghkJzJY07hj6mP94sS9+naT77+xLjc9S371hFxIROTMGMHYS4OuNeY+0Q7dmwWY/gPWVe2lQK0QGVifR9mnZEN+/EI9n45viJ41kXO0cmFPzk8yuZ0LLhujXKgTj+0bjyBsP6O1mKiqrMnqMmgUDht+dMq0/ZiTFYvoDrc1u5ZkzrA0a1vPRmaNHuwtJlRBrzlHNCTL6xITgr5f7oWOTILPqCQBtI2QWTbBnaIVwma83uusZpWVqqL6/D1PgiMh51ekAZvHixWjevDl8fX0RHx+PgwcPOrpKRhmaDff7F3riMz2tLdrG9tH9xi718sCqSQmY/kBrLHqqM5o29Me7j3dAtEb3hHYLjCUPJk8PCX4YF485D7U1GCTISyv1blcVNzxTr/jhbOk3/lZhAUgeGGO0ZaXe3aTi0QnVrVbj+7XA4TmJGNunuaicUqjucrlXN+M013cyp4VL89pG9IhC4yA/PGZijpeV/2d8MsO+MeKA1tyuNBVjI+EAYMr9MWjfWIY3H675SDEiotpWZ7+CrVy5EtOnT8fSpUsRHx+PTz75BElJScjMzERoaKijq6fXV2O6690ukUjwUMcI5MrL0C5S/xTuCS0ailpSguv5IP9OBQa1CUWTBv54aVArnWOqGMuBaR1a38Kr0PVcr2b4LvUSBrcNE22vpxEo9Y0Jwe2SCrRoZNn5zB1GfF/rRli276LOIpMvDWqFQW1C0SLk3nklEgkGxIZiyciumLz8iHq7Zn5K8d1WJWOJ0aMTmuHCjTvo1qyByfppDnF+b3hHCIJgtNVofN9oBPh6G73671/oiZJKBX5Pv4rdf9/EU9115+AxpspE119wPR+se7Hur2NFRKRPnQ1gPvroI0yYMAFjx44FACxduhTr16/HN998g9dee83BtdOvcZDhNZIkEgnG99Ndw2bdi33xfepFTH9AnIC6e+ZA3CquQFMDCZoeEkDm6wV5WZXeMmv+2Rv7zt8ynE8D8xN4W4UF4Pi8wer1m+Y93Bb7L+Tjkc6R6mv7YVxPCIL+bo6Q+j64WVyB+OiG2Pn3DfX2Ie3C4eVhXiPggNhG+GlCL7QM1U2MjQnVP0fMUK1RSZpJxKYWZZQAmP+o/mRqfbTnaFEFL68NjcMf6dequ7AkEnw9pjt2n72BRztXt840MfJvxsNDgvpSL4yMb4aR8YY/R4Ncf81RInJjdTKAqaioQFpaGmbPnq3e5uHhgcTERKSmpup9T3l5OcrLy9Wv5XK53esJAKEBUuQVlZsuaED7xoFY+KTu8NZ6Ui9Rl4c2iUSCg68nQikIkHp5YnRCc8z89Tj6xFTPtNulaQN0aWq85UBqZP4YbTLfe60Xz/eJxvNa3V0SiW4rUEyj6sBi9eQ++CXtMp7v3Rzd3tkCoDpXRVW/5ePjMfKrA0bPL5FIkKA1izAARAUbH4HTpWkQjmYXILFNdevRtlfug7ysCqFGJv8zZd4j7fDyz+n4v/taQOrpgW/2XsQrg1vrLTvpvpaYdF/Lu5MXVgclmkOlXxzUCoWllRjW0fAQcGv0bB6MntGmZy8mInJWEsGek1xY6dq1a2jcuDH27duHhIR7oytmzpyJnTt34sAB3YfdvHnz8NZbb+lsLywshExm25WaNZ3JkWPeHxl4dXCs3kTK2iIIAjJzi9AipL7OcGVty/Zm4Yf9l/Dj+HhEBBpfWdsaWTfvIP9OObo1070fv6VdwYmrhZj7UFtRa82ao1cw89fjWPpcNwxqE6bzPm17z91E+uUC/HNAS6NdNfl3KvDXiet4uFOkqAtJJe1SPoYvqQ6Kf5rQC/9cnobbJZUYk9AMbxlpgbl9pwIN6lUnySqUQp1YkbvDvE0oKqvCB092xD+6Rzm6OkREVpHL5QgMDDT5/HaZAEZfC0xUVJTdAxiynSqFEl4WtArZyt+5RfDz9kRUsD+uFZRi65k8PNm1iclZh+uaG0XlOHmtEPe1amRwxBIRUV1nbgBTJ7uQQkJC4OnpidzcXNH23NxchIfrn01WKpVCKjWe10B1myOCFwBoHXYvhyYyyA+jjOQN1WWNAqQYGFs3E9yJiGytTg6j9vHxQbdu3bB161b1NqVSia1bt4paZIiIiMg91ckWGACYPn06xowZg+7du6Nnz5745JNPcOfOHfWoJCIiInJfdTaAefrpp3Hjxg3MnTsXOTk56Ny5MzZu3IiwMNMJnkREROTa6mQSry2YmwREREREdYe5z+86mQNDREREZAwDGCIiInI6DGCIiIjI6TCAISIiIqfDAIaIiIicDgMYIiIicjoMYIiIiMjpMIAhIiIip8MAhoiIiJxOnV1KoKZUEwzL5XIH14SIiIjMpXpum1oowGUDmKKiIgBAVFSUg2tCRERElioqKkJgYKDB/S67FpJSqcS1a9cQEBAAiURis+PK5XJERUXh8uXLXGPJznivawfvc+3gfa4dvM+1w573WRAEFBUVITIyEh4ehjNdXLYFxsPDA02aNLHb8WUyGf/nqCW817WD97l28D7XDt7n2mGv+2ys5UWFSbxERETkdBjAEBERkdNhAGMhqVSKN998E1Kp1NFVcXm817WD97l28D7XDt7n2lEX7rPLJvESERGR62ILDBERETkdBjBERETkdBjAEBERkdNhAENEREROhwGMhRYvXozmzZvD19cX8fHxOHjwoKOr5FTmzZsHiUQi+omLi1PvLysrQ3JyMho2bIj69etj+PDhyM3NFR0jOzsbw4YNg7+/P0JDQzFjxgxUVVXV9qXUKbt27cLDDz+MyMhISCQSrF27VrRfEATMnTsXERER8PPzQ2JiIs6ePSsqk5+fj5EjR0ImkyEoKAjjxo1DcXGxqMzx48fRr18/+Pr6IioqCgsXLrT3pdUppu7z888/r/Pve8iQIaIyvM+mLViwAD169EBAQABCQ0Px2GOPITMzU1TGVn8rduzYga5du0IqlSImJgbLli2z9+XVGebc5wEDBuj8m540aZKojMPus0Bm+/nnnwUfHx/hm2++ETIyMoQJEyYIQUFBQm5urqOr5jTefPNNoV27dsL169fVPzdu3FDvnzRpkhAVFSVs3bpVOHz4sNCrVy+hd+/e6v1VVVVC+/bthcTEROHo0aPCX3/9JYSEhAizZ892xOXUGX/99Zfw+uuvC6tXrxYACGvWrBHtf++994TAwEBh7dq1wrFjx4RHHnlEiI6OFkpLS9VlhgwZInTq1EnYv3+/sHv3biEmJkZ45pln1PsLCwuFsLAwYeTIkcLJkyeFn376SfDz8xO++OKL2rpMhzN1n8eMGSMMGTJE9O87Pz9fVIb32bSkpCTh22+/FU6ePCmkp6cLDz74oNC0aVOhuLhYXcYWfysuXLgg+Pv7C9OnTxdOnTolfPbZZ4Knp6ewcePGWr1eRzHnPt93333ChAkTRP+mCwsL1fsdeZ8ZwFigZ8+eQnJysvq1QqEQIiMjhQULFjiwVs7lzTffFDp16qR3X0FBgeDt7S388ssv6m2nT58WAAipqamCIFQ/QDw8PIScnBx1mSVLlggymUwoLy+3a92dhfaDValUCuHh4cIHH3yg3lZQUCBIpVLhp59+EgRBEE6dOiUAEA4dOqQus2HDBkEikQhXr14VBEEQPv/8c6FBgwai+zxr1iwhNjbWzldUNxkKYB599FGD7+F9tk5eXp4AQNi5c6cgCLb7WzFz5kyhXbt2onM9/fTTQlJSkr0vqU7Svs+CUB3AvPzyywbf48j7zC4kM1VUVCAtLQ2JiYnqbR4eHkhMTERqaqoDa+Z8zp49i8jISLRo0QIjR45EdnY2ACAtLQ2VlZWiexwXF4emTZuq73Fqaio6dOiAsLAwdZmkpCTI5XJkZGTU7oU4iaysLOTk5Ijua2BgIOLj40X3NSgoCN27d1eXSUxMhIeHBw4cOKAu079/f/j4+KjLJCUlITMzE7dv366lq6n7duzYgdDQUMTGxmLy5Mm4deuWeh/vs3UKCwsBAMHBwQBs97ciNTVVdAxVGXf9m659n1WWL1+OkJAQtG/fHrNnz0ZJSYl6nyPvs8su5mhrN2/ehEKhEH1IABAWFoYzZ844qFbOJz4+HsuWLUNsbCyuX7+Ot956C/369cPJkyeRk5MDHx8fBAUFid4TFhaGnJwcAEBOTo7ez0C1j3Sp7ou++6Z5X0NDQ0X7vby8EBwcLCoTHR2tcwzVvgYNGtil/s5kyJAheOKJJxAdHY3z58/jX//6F4YOHYrU1FR4enryPltBqVRi6tSp6NOnD9q3bw8ANvtbYaiMXC5HaWkp/Pz87HFJdZK++wwAzz77LJo1a4bIyEgcP34cs2bNQmZmJlavXg3AsfeZAQzVqqFDh6p/79ixI+Lj49GsWTOsWrXKrf5YkGsaMWKE+vcOHTqgY8eOaNmyJXbs2IFBgwY5sGbOKzk5GSdPnsSePXscXRWXZug+T5w4Uf17hw4dEBERgUGDBuH8+fNo2bJlbVdThF1IZgoJCYGnp6dOlntubi7Cw8MdVCvnFxQUhNatW+PcuXMIDw9HRUUFCgoKRGU073F4eLjez0C1j3Sp7ouxf7vh4eHIy8sT7a+qqkJ+fj7vfQ20aNECISEhOHfuHADeZ0tNmTIF69atw/bt29GkSRP1dlv9rTBURiaTudUXKkP3WZ/4+HgAEP2bdtR9ZgBjJh8fH3Tr1g1bt25Vb1Mqldi6dSsSEhIcWDPnVlxcjPPnzyMiIgLdunWDt7e36B5nZmYiOztbfY8TEhJw4sQJ0UMgJSUFMpkMbdu2rfX6O4Po6GiEh4eL7qtcLseBAwdE97WgoABpaWnqMtu2bYNSqVT/wUpISMCuXbtQWVmpLpOSkoLY2Fi369Yw15UrV3Dr1i1EREQA4H02lyAImDJlCtasWYNt27bpdKnZ6m9FQkKC6BiqMu7yN93UfdYnPT0dAET/ph12n2uUAuxmfv75Z0EqlQrLli0TTp06JUycOFEICgoSZV+Tca+88oqwY8cOISsrS9i7d6+QmJgohISECHl5eYIgVA+NbNq0qbBt2zbh8OHDQkJCgpCQkKB+v2rI3uDBg4X09HRh48aNQqNGjdx+GHVRUZFw9OhR4ejRowIA4aOPPhKOHj0qXLp0SRCE6mHUQUFBwu+//y4cP35cePTRR/UOo+7SpYtw4MABYc+ePUKrVq1Ew3sLCgqEsLAwYdSoUcLJkyeFn3/+WfD393er4b3G7nNRUZHw6quvCqmpqUJWVpawZcsWoWvXrkKrVq2EsrIy9TF4n02bPHmyEBgYKOzYsUM0fLekpERdxhZ/K1TDe2fMmCGcPn1aWLx4sVsNozZ1n8+dOyfMnz9fOHz4sJCVlSX8/vvvQosWLYT+/furj+HI+8wAxkKfffaZ0LRpU8HHx0fo2bOnsH//fkdXyak8/fTTQkREhODj4yM0btxYePrpp4Vz586p95eWlgr//Oc/hQYNGgj+/v7C448/Lly/fl10jIsXLwpDhw4V/Pz8hJCQEOGVV14RKisra/tS6pTt27cLAHR+xowZIwhC9VDqN954QwgLCxOkUqkwaNAgITMzU3SMW7duCc8884xQv359QSaTCWPHjhWKiopEZY4dOyb07dtXkEqlQuPGjYX33nuvti6xTjB2n0tKSoTBgwcLjRo1Ery9vYVmzZoJEyZM0PmCw/tsmr57DED49ttv1WVs9bdi+/btQufOnQUfHx+hRYsWonO4OlP3OTs7W+jfv78QHBwsSKVSISYmRpgxY4ZoHhhBcNx9lty9CCIiIiKnwRwYIiIicjoMYIiIiMjpMIAhIiIip8MAhoiIiJwOAxgiIiJyOgxgiIiIyOkwgCEiIiKnwwCGiIiInA4DGCIiInI6DGCIiIjI6TCAISIiIqfDAIaIiIiczv8D0fQTfkg5spsAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# We also need a sampler for the training set. There are three\n",
    "# special samplers in the `wids` package that work particularly\n",
    "# well with sharded datasets:\n",
    "# - `wids.ShardedSampler` shuffles shards and then samples in shards;\n",
    "#   it guarantees that only one shard is used at a time\n",
    "# - `wids.ChunkedSampler` samples by fixed sized chunks, shuffles\n",
    "#   the chunks, and the the samples within each chunk\n",
    "# - `wids.DistributedChunkedSampler` is like `ChunkedSampler` but\n",
    "#   works with distributed training (it first divides the entire\n",
    "#   dataset into per-node chunks, then the per-node chunks into\n",
    "#   smaller chunks, then shuffles the smaller chunks)\n",
    "\n",
    "# trainsampler = wids.ShardedSampler(trainset)\n",
    "# trainsampler = wids.ChunkedSampler(trainset, chunksize=1000, shuffle=True)\n",
    "trainsampler = wids.DistributedChunkedSampler(trainset, chunksize=1000, shuffle=True)\n",
    "\n",
    "plt.plot(list(trainsampler)[:2500])\n",
    "\n",
    "# Note that the sampler shuffles within each shard before moving on to\n",
    "# the next shard. Furthermore, on the first epoch, the sampler\n",
    "# uses the shards in order, but on subsequent epochs, it shuffles\n",
    "# them. This makes testing and debugging easier. If you don't like\n",
    "# this behavior, you can use shufflefirst=True\n",
    "\n",
    "trainsampler.set_epoch(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "17890256",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T08:07:09.543317Z",
     "iopub.status.busy": "2023-12-18T08:07:09.543099Z",
     "iopub.status.idle": "2023-12-18T08:07:10.091435Z",
     "shell.execute_reply": "2023-12-18T08:07:10.090728Z"
    },
    "papermill": {
     "duration": 0.555114,
     "end_time": "2023-12-18T08:07:10.093942",
     "exception": false,
     "start_time": "2023-12-18T08:07:09.538828",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 3, 224, 224]) torch.Size([32])\n"
     ]
    }
   ],
   "source": [
    "# Create data loaders for the training and validation datasets\n",
    "\n",
    "trainloader = DataLoader(trainset, batch_size=batch_size, num_workers=4, sampler=trainsampler)\n",
    "valloader = DataLoader(valset, batch_size=batch_size, shuffle=False, num_workers=4)\n",
    "\n",
    "images, classes = next(iter(trainloader))\n",
    "print(images.shape, classes.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6771e961",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T08:07:10.108464Z",
     "iopub.status.busy": "2023-12-18T08:07:10.108265Z",
     "iopub.status.idle": "2023-12-18T08:07:12.114638Z",
     "shell.execute_reply": "2023-12-18T08:07:12.113645Z"
    },
    "papermill": {
     "duration": 2.017025,
     "end_time": "2023-12-18T08:07:12.117869",
     "exception": false,
     "start_time": "2023-12-18T08:07:10.100844",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tmb/proj/webdataset/venv/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
      "  warnings.warn(\n",
      "/home/tmb/proj/webdataset/venv/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=None`.\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "# The usual PyTorch model definition. We use an uninitialized ResNet50 model.\n",
    "\n",
    "model = resnet50(pretrained=False)\n",
    "\n",
    "# Define the loss function and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4)\n",
    "\n",
    "# Move the model to the GPU if available\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2c03980d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T08:07:12.135168Z",
     "iopub.status.busy": "2023-12-18T08:07:12.134833Z",
     "iopub.status.idle": "2023-12-18T08:08:17.720615Z",
     "shell.execute_reply": "2023-12-18T08:08:17.719937Z"
    },
    "papermill": {
     "duration": 65.595997,
     "end_time": "2023-12-18T08:08:17.722244",
     "exception": false,
     "start_time": "2023-12-18T08:07:12.126247",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,    18] loss: 4.92214 correct: 0.06597\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,    44] loss: 4.47568 correct: 0.07315\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,    70] loss: 4.05072 correct: 0.09420\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,    95] loss: 3.76577 correct: 0.11086\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,   120] loss: 3.29646 correct: 0.14625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,   146] loss: 2.90517 correct: 0.18688\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,   171] loss: 2.69376 correct: 0.21656\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,   196] loss: 2.50486 correct: 0.26656\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,   221] loss: 2.35720 correct: 0.30719\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,   246] loss: 2.22152 correct: 0.34719\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,   271] loss: 2.09821 correct: 0.38562\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,   296] loss: 2.01367 correct: 0.39969\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "losses, accuracies = deque(maxlen=100), deque(maxlen=100)\n",
    "\n",
    "steps = 0\n",
    "\n",
    "# Train the model\n",
    "for epoch in range(epochs):\n",
    "    for i, data, verbose in enumerate_report(trainloader, 5):\n",
    "        # get the inputs; data is a list of [inputs, labels]\n",
    "        inputs, labels = data[0].to(device), data[1].to(device)\n",
    "\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = model(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        pred = outputs.cpu().detach().argmax(dim=1, keepdim=True)\n",
    "        correct = pred.eq(labels.cpu().view_as(pred)).sum().item()\n",
    "        accuracy = correct / float(len(labels))\n",
    "\n",
    "        losses.append(loss.item())\n",
    "        accuracies.append(accuracy)\n",
    "        steps += len(labels)\n",
    "\n",
    "        if verbose and len(losses) > 5:\n",
    "            print(\n",
    "                \"[%d, %5d] loss: %.5f correct: %.5f\"\n",
    "                % (epoch + 1, i + 1, np.mean(losses), np.mean(accuracies))\n",
    "            )\n",
    "            running_loss = 0.0\n",
    "\n",
    "        if steps > max_steps:\n",
    "            break\n",
    "    if steps > max_steps:\n",
    "        break\n",
    "\n",
    "print(\"Finished Training\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 73.560731,
   "end_time": "2023-12-18T08:08:18.846571",
   "environment_variables": {},
   "exception": null,
   "input_path": "train-resnet50-wids.ipynb",
   "output_path": "out/_train-resnet50-wids.ipynb",
   "parameters": {
    "max_steps": 10000
   },
   "start_time": "2023-12-18T08:07:05.285840",
   "version": "2.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}