MetaSDF.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WTwikOyWZobA"
   },
   "source": [
    "# MetaSDF & Meta-SIREN\n",
    "\n",
    "This is a colab to explore MetaSDF, and its applications to rapidly fit neural implicit representations.\n",
    "\n",
    "Make sure to switch the runtime type to \"GPU\" under \"Runtime --> Change Runtime Type\"!\n",
    "\n",
    "We will show you how to run two experiments using gradient-based meta-learning: \n",
    "* [Fitting an image in 3 gradient descent steps with SIREN](#section_1)\n",
    "* [Fitting 2D Signed Distance Functions of MNIST digits](#section_2)\n",
    "\n",
    "Let's go! \n",
    "\n",
    "First, the imports:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "eJ49alfvZobQ"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torchvision\n",
    "from torchvision import transforms\n",
    "import scipy.ndimage\n",
    "from torch import nn\n",
    "from collections import OrderedDict, Mapping\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "from torch.nn.init import _calculate_correct_fan"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "e2D7783s2tg2"
   },
   "source": [
    "For meta-learning, we're using the excellent \"Torchmeta\" library. We have to install it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "eX_d5hsYZtSq",
    "outputId": "9ab3ba4f-f984-4e02-f792-6e1a28b55e25"
   },
   "outputs": [],
   "source": [
    "!pip install torchmeta\n",
    "from torchmeta.modules import (MetaModule, MetaSequential, MetaLinear)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Je35HXUPZobV"
   },
   "source": [
    "We're now ready to implement a few neural network layers: Fully connected networks, and SIREN."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DlEcUhGiZobX",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class BatchLinear(nn.Linear, MetaModule):\n",
    "    '''A linear meta-layer that can deal with batched weight matrices and biases, as for instance output by a\n",
    "    hypernetwork.'''\n",
    "    __doc__ = nn.Linear.__doc__\n",
    "\n",
    "    def forward(self, input, params=None):\n",
    "        if params is None:\n",
    "            params = OrderedDict(self.named_parameters())\n",
    "\n",
    "        bias = params.get('bias', None)\n",
    "        weight = params['weight']\n",
    "\n",
    "        output = input.matmul(weight.permute(*[i for i in range(len(weight.shape)-2)], -1, -2))\n",
    "        output += bias.unsqueeze(-2)\n",
    "        return output\n",
    "\n",
    "\n",
    "class MetaFC(MetaModule):\n",
    "    '''A fully connected neural network that allows swapping out the weights, either via a hypernetwork\n",
    "    or via MAML.\n",
    "    '''\n",
    "    def __init__(self, in_features, out_features,\n",
    "                 num_hidden_layers, hidden_features,\n",
    "                 outermost_linear=False):\n",
    "        super().__init__()\n",
    "\n",
    "        self.net = []\n",
    "        self.net.append(MetaSequential(\n",
    "            BatchLinear(in_features, hidden_features),\n",
    "            nn.ReLU(inplace=True)\n",
    "        ))\n",
    "\n",
    "        for i in range(num_hidden_layers):\n",
    "            self.net.append(MetaSequential(\n",
    "                BatchLinear(hidden_features, hidden_features),\n",
    "                nn.ReLU(inplace=True)\n",
    "            ))\n",
    "\n",
    "        if outermost_linear:\n",
    "            self.net.append(MetaSequential(\n",
    "                BatchLinear(hidden_features, out_features),\n",
    "            ))\n",
    "        else:\n",
    "            self.net.append(MetaSequential(\n",
    "                BatchLinear(hidden_features, out_features),\n",
    "                nn.ReLU(inplace=True)\n",
    "            ))\n",
    "\n",
    "        self.net = MetaSequential(*self.net)\n",
    "        self.net.apply(init_weights_normal)\n",
    "\n",
    "    def forward(self, coords, params=None, **kwargs):\n",
    "        '''Simple forward pass without computation of spatial gradients.'''\n",
    "        output = self.net(coords, params=self.get_subdict(params, 'net'))\n",
    "        return output\n",
    "\n",
    "\n",
    "class SineLayer(MetaModule):\n",
    "    # See paper sec. 3.2, final paragraph, and supplement Sec. 1.5 for discussion of omega_0.\n",
    "\n",
    "    # If is_first=True, omega_0 is a frequency factor which simply multiplies the activations before the\n",
    "    # nonlinearity. Different signals may require different omega_0 in the first layer - this is a\n",
    "    # hyperparameter.\n",
    "\n",
    "    # If is_first=False, then the weights will be divided by omega_0 so as to keep the magnitude of\n",
    "    # activations constant, but boost gradients to the weight matrix (see supplement Sec. 1.5)\n",
    "\n",
    "    def __init__(self, in_features, out_features, bias=True, is_first=False, omega_0=30):\n",
    "        super().__init__()\n",
    "        self.omega_0 = float(omega_0)\n",
    "\n",
    "        self.is_first = is_first\n",
    "\n",
    "        self.in_features = in_features\n",
    "        self.linear = BatchLinear(in_features, out_features, bias=bias)\n",
    "        self.init_weights()\n",
    "\n",
    "    def init_weights(self):\n",
    "        with torch.no_grad():\n",
    "            if self.is_first:\n",
    "                self.linear.weight.uniform_(-1 / self.in_features,\n",
    "                                            1 / self.in_features)\n",
    "            else:\n",
    "                self.linear.weight.uniform_(-np.sqrt(6 / self.in_features) / self.omega_0,\n",
    "                                            np.sqrt(6 / self.in_features) / self.omega_0)\n",
    "\n",
    "    def forward(self, input, params=None):\n",
    "        intermed = self.linear(input, params=self.get_subdict(params, 'linear'))\n",
    "        return torch.sin(self.omega_0 * intermed)\n",
    "\n",
    "\n",
    "class Siren(MetaModule):\n",
    "    def __init__(self, in_features, hidden_features, hidden_layers, out_features, outermost_linear=False,\n",
    "                 first_omega_0=30, hidden_omega_0=30., special_first=True):\n",
    "        super().__init__()\n",
    "        self.hidden_omega_0 = hidden_omega_0\n",
    "\n",
    "        layer = SineLayer\n",
    "\n",
    "        self.net = []\n",
    "        self.net.append(layer(in_features, hidden_features,\n",
    "                              is_first=special_first, omega_0=first_omega_0))\n",
    "\n",
    "        for i in range(hidden_layers):\n",
    "            self.net.append(layer(hidden_features, hidden_features,\n",
    "                                  is_first=False, omega_0=hidden_omega_0))\n",
    "\n",
    "        if outermost_linear:\n",
    "            final_linear = BatchLinear(hidden_features, out_features)\n",
    "\n",
    "            with torch.no_grad():\n",
    "                final_linear.weight.uniform_(-np.sqrt(6 / hidden_features) / 30.,\n",
    "                                             np.sqrt(6 / hidden_features) / 30.)\n",
    "            self.net.append(final_linear)\n",
    "        else:\n",
    "            self.net.append(layer(hidden_features, out_features, is_first=False, omega_0=hidden_omega_0))\n",
    "\n",
    "        self.net = nn.ModuleList(self.net)\n",
    "\n",
    "    def forward(self, coords, params=None):\n",
    "        x = coords\n",
    "\n",
    "        for i, layer in enumerate(self.net):\n",
    "            x = layer(x, params=self.get_subdict(params, f'net.{i}'))\n",
    "\n",
    "        return x\n",
    "    \n",
    "    \n",
    "def init_weights_normal(m):\n",
    "    if type(m) == BatchLinear or nn.Linear:\n",
    "        if hasattr(m, 'weight'):\n",
    "            torch.nn.init.kaiming_normal_(m.weight, nonlinearity='relu')\n",
    "        if hasattr(m, 'bias'):\n",
    "            m.bias.data.fill_(0.)\n",
    "            \n",
    "            \n",
    "def get_mgrid(sidelen):\n",
    "    # Generate 2D pixel coordinates from an image of sidelen x sidelen\n",
    "    pixel_coords = np.stack(np.mgrid[:sidelen,:sidelen], axis=-1)[None,...].astype(np.float32)\n",
    "    pixel_coords /= sidelen    \n",
    "    pixel_coords -= 0.5\n",
    "    pixel_coords = torch.Tensor(pixel_coords).view(-1, 2)\n",
    "    return pixel_coords"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0cPO4PWDZobZ"
   },
   "source": [
    "Now, we implement MAML. The important parts of the code are commented, so it's easy to understand how each part works! Start by looking at the \"forward\" function.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "OL4dgzRiZoba",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def l2_loss(prediction, gt):\n",
    "    return ((prediction - gt)**2).mean()\n",
    "\n",
    "\n",
    "class MAML(nn.Module):\n",
    "    def __init__(self, num_meta_steps, hypo_module, loss, init_lr,\n",
    "                 lr_type='static', first_order=False):\n",
    "        super().__init__()\n",
    "\n",
    "        self.hypo_module = hypo_module # The module who's weights we want to meta-learn.\n",
    "        self.first_order = first_order\n",
    "        self.loss = loss\n",
    "        self.lr_type = lr_type\n",
    "        self.log = []\n",
    "\n",
    "        self.register_buffer('num_meta_steps', torch.Tensor([num_meta_steps]).int())\n",
    "\n",
    "        if self.lr_type == 'static': \n",
    "            self.register_buffer('lr', torch.Tensor([init_lr]))\n",
    "        elif self.lr_type == 'global':\n",
    "            self.lr = nn.Parameter(torch.Tensor([init_lr]))\n",
    "        elif self.lr_type == 'per_step':\n",
    "            self.lr = nn.ParameterList([nn.Parameter(torch.Tensor([init_lr]))\n",
    "                                        for _ in range(num_meta_steps)])\n",
    "        elif self.lr_type == 'per_parameter': # As proposed in \"Meta-SGD\".\n",
    "            self.lr = nn.ParameterList([])\n",
    "            hypo_parameters = hypo_module.parameters()\n",
    "            for param in hypo_parameters:\n",
    "                self.lr.append(nn.Parameter(torch.ones(param.size()) * init_lr))\n",
    "        elif self.lr_type == 'per_parameter_per_step':\n",
    "            self.lr = nn.ModuleList([])\n",
    "            for name, param in hypo_module.meta_named_parameters():\n",
    "                self.lr.append(nn.ParameterList([nn.Parameter(torch.ones(param.size()) * init_lr)\n",
    "                                                 for _ in range(num_meta_steps)]))\n",
    "\n",
    "        param_count = 0\n",
    "        for param in self.parameters():\n",
    "            param_count += np.prod(param.shape)\n",
    "\n",
    "        print(param_count)\n",
    "\n",
    "    def _update_step(self, loss, param_dict, step):\n",
    "        grads = torch.autograd.grad(loss, param_dict.values(),\n",
    "                                    create_graph=False if self.first_order else True)\n",
    "        params = OrderedDict()\n",
    "        for i, ((name, param), grad) in enumerate(zip(param_dict.items(), grads)):\n",
    "            if self.lr_type in ['static', 'global']:\n",
    "                lr = self.lr\n",
    "                params[name] = param - lr * grad\n",
    "            elif self.lr_type in ['per_step']:\n",
    "                lr = self.lr[step]\n",
    "                params[name] = param - lr * grad\n",
    "            elif self.lr_type in ['per_parameter']:\n",
    "                lr = self.lr[i]\n",
    "                params[name] = param - lr * grad\n",
    "            elif self.lr_type in ['per_parameter_per_step']:\n",
    "                lr = self.lr[i][step]\n",
    "                params[name] = param - lr * grad\n",
    "            else:\n",
    "                raise NotImplementedError\n",
    "\n",
    "        return params, grads\n",
    "\n",
    "    def forward_with_params(self, query_x, fast_params, **kwargs):\n",
    "        output = self.hypo_module(query_x, params=fast_params)\n",
    "        return output\n",
    "\n",
    "    def generate_params(self, context_dict):\n",
    "        \"\"\"Specializes the model\"\"\"\n",
    "        x = context_dict.get('x').cuda()\n",
    "        y = context_dict.get('y').cuda()\n",
    "\n",
    "        meta_batch_size = x.shape[0]\n",
    "\n",
    "        with torch.enable_grad():\n",
    "            # First, replicate the initialization for each batch item.\n",
    "            # This is the learned initialization, i.e., in the outer loop,\n",
    "            # the gradients are backpropagated all the way into the \n",
    "            # \"meta_named_parameters\" of the hypo_module.\n",
    "            fast_params = OrderedDict()\n",
    "            for name, param in self.hypo_module.meta_named_parameters():\n",
    "                fast_params[name] = param[None, ...].repeat((meta_batch_size,) + (1,) * len(param.shape))\n",
    "\n",
    "            prev_loss = 1e6\n",
    "            intermed_predictions = []\n",
    "            for j in range(self.num_meta_steps):\n",
    "                # Using the current set of parameters, perform a forward pass with the context inputs.\n",
    "                predictions = self.hypo_module(x, params=fast_params)\n",
    "\n",
    "                # Compute the loss on the context labels.\n",
    "                loss = self.loss(predictions, y)\n",
    "                intermed_predictions.append(predictions)\n",
    "\n",
    "                if loss > prev_loss:\n",
    "                    print('inner lr too high?')\n",
    "                \n",
    "                # Using the computed loss, update the fast parameters.\n",
    "                fast_params, grads = self._update_step(loss, fast_params, j)\n",
    "                prev_loss = loss\n",
    "\n",
    "        return fast_params, intermed_predictions\n",
    "\n",
    "    def forward(self, meta_batch, **kwargs):\n",
    "        # The meta_batch conists of the \"context\" set (the observations we're conditioning on)\n",
    "        # and the \"query\" inputs (the points where we want to evaluate the specialized model)\n",
    "        context = meta_batch['context']\n",
    "        query_x = meta_batch['query']['x'].cuda()\n",
    "\n",
    "        # Specialize the model with the \"generate_params\" function.\n",
    "        fast_params, intermed_predictions = self.generate_params(context)\n",
    "\n",
    "        # Compute the final outputs.\n",
    "        model_output = self.hypo_module(query_x, params=fast_params)\n",
    "        out_dict = {'model_out':model_output, 'intermed_predictions':intermed_predictions}\n",
    "\n",
    "        return out_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9i0VRikaZobc"
   },
   "source": [
    "<a id='section_1'></a>\n",
    "## Learning to fit images in 3 gradient descent steps\n",
    "\n",
    "By learning an initialization for SIREN, we may fit any image in as few as 3 gradient descent steps! \n",
    "This has also been noted by Tancik et al. in \"Learned Initializations for Optimizing Coordinate-Based Neural Representations\" (2020).\n",
    "\n",
    "We'll demonstrate here with Cifar-10, but it works just as well with CelebA or imagenet - try it out yourself!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gKFj5-FVZobc",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class CIFAR10():\n",
    "    def __init__(self):\n",
    "        transform = transforms.Compose(\n",
    "            [transforms.ToTensor(),\n",
    "             transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "\n",
    "        self.dataset = torchvision.datasets.CIFAR10(root='./data', train=True,\n",
    "                                                download=True, transform=transform)\n",
    "        \n",
    "        self.meshgrid = get_mgrid(sidelen=32)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.dataset)\n",
    "        \n",
    "    def __getitem__(self, item):\n",
    "        img, _ = self.dataset[item]\n",
    "        img_flat = img.permute(1,2,0).view(-1, 3)\n",
    "        return {'context':{'x':self.meshgrid, 'y':img_flat}, \n",
    "                'query':{'x':self.meshgrid, 'y':img_flat}}\n",
    "\n",
    "\n",
    "def lin2img(tensor):\n",
    "    batch_size, num_samples, channels = tensor.shape\n",
    "    sidelen = np.sqrt(num_samples).astype(int)\n",
    "    return tensor.view(batch_size, sidelen, sidelen, channels).squeeze(-1)\n",
    "\n",
    "    \n",
    "def plot_sample_image(img_batch, ax):\n",
    "    img = lin2img(img_batch)[0].detach().cpu().numpy()\n",
    "    img += 1\n",
    "    img /= 2.\n",
    "    img = np.clip(img, 0., 1.)\n",
    "    ax.set_axis_off()\n",
    "    ax.imshow(img)\n",
    "\n",
    "\n",
    "def dict_to_gpu(ob):\n",
    "    if isinstance(ob, Mapping):\n",
    "        return {k: dict_to_gpu(v) for k, v in ob.items()}\n",
    "    else:\n",
    "        return ob.cuda()    \n",
    "\n",
    "\n",
    "def dict_to_gpu(ob):\n",
    "    if isinstance(ob, Mapping):\n",
    "        return {k: dict_to_gpu(v) for k, v in ob.items()}\n",
    "    else:\n",
    "        return ob.cuda()    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9t9tBE4EZobd"
   },
   "source": [
    "Now, let's initialize our models and our dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 140,
     "referenced_widgets": [
      "4098608467004bec96d3b6708c5e0ab0",
      "6af97e126e0f401da67392f76628967a",
      "5d85b038e8c74f59a10ddec2cf7e7f8d",
      "484a8d36c5174d619cf4492d5c2fc1c3",
      "e1534d9189f84e56bff73e88ecc5913a",
      "814d88b570274efc871a7fcff975ae11",
      "fe8901d5c9484ae59f5d0ba7e851d6b4",
      "24ac3163a23949079bef466ca70e4bec"
     ]
    },
    "id": "tKLBYi25Zobd",
    "outputId": "670ef5dc-6ed4-4412-988f-f3fc9a638407",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "img_siren = Siren(in_features=2, hidden_features=128, hidden_layers=3, out_features=3, outermost_linear=True).cuda()\n",
    "meta_siren = MAML(num_meta_steps=3, hypo_module=img_siren, loss=l2_loss, init_lr=1e-5, \n",
    "                  lr_type='per_parameter_per_step').cuda()\n",
    "\n",
    "dataset = CIFAR10()\n",
    "dataloader = DataLoader(dataset, batch_size=32, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WS7Juv9HZobf"
   },
   "source": [
    "Let's train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "pzmwE0_KZobg",
    "outputId": "7505bb5e-9b52-4a70-8ae1-3c53379e26f4",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "steps_til_summary = 100\n",
    "\n",
    "optim = torch.optim.Adam(lr=5e-5, params=meta_siren.parameters())\n",
    "\n",
    "for step, sample in enumerate(dataloader):\n",
    "    sample = dict_to_gpu(sample)\n",
    "    model_output = meta_siren(sample)    \n",
    "    loss = ((model_output['model_out'] - sample['query']['y'])**2).mean()\n",
    "    \n",
    "    if not step % steps_til_summary:\n",
    "        print(\"Step %d, Total loss %0.6f\" % (step, loss))\n",
    "        fig, axes = plt.subplots(1,5, figsize=(30,6))\n",
    "        ax_titles = ['Learned Initialization', 'Inner step 1 output', \n",
    "                     'Inner step 2 output', 'Inner step 3 output', \n",
    "                     'Ground Truth']\n",
    "        for i, inner_step_out in enumerate(model_output['intermed_predictions']):\n",
    "            plot_sample_image(inner_step_out, ax=axes[i])\n",
    "            axes[i].set_title(ax_titles[i], fontsize=25)\n",
    "        plot_sample_image(model_output['model_out'], ax=axes[-2])\n",
    "        axes[-2].set_title(ax_titles[-2], fontsize=25)\n",
    "\n",
    "        plot_sample_image(sample['query']['y'], ax=axes[-1])\n",
    "        axes[-1].set_title(ax_titles[-1], fontsize=25)\n",
    "\n",
    "        plt.show()\n",
    "\n",
    "    optim.zero_grad()\n",
    "    loss.backward()\n",
    "    optim.step()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iT1Qy7983EbD"
   },
   "source": [
    "As you can see, after a few hundred steps of training, we can fit any of the Cifar-10 images in only three gradient descent steps!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L7E8yAAIZobg"
   },
   "source": [
    "<a id='section_2'></a>\n",
    "## Learning to fit 2D MNIST SDFs\n",
    "\n",
    "Now, we will see how we can fit 2D SDFs quickly using gradient-based meta-learning.\n",
    "\n",
    "We will use MNIST as an example, transforming them into 2D signed distance fields, where the contour is the zero-level set.\n",
    "\n",
    "First, the dataset code:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mCB6SHXGZobh"
   },
   "outputs": [],
   "source": [
    "class SignedDistanceTransform:\n",
    "    def __call__(self, img_tensor):\n",
    "        # Threshold.\n",
    "        img_tensor[img_tensor<0.5] = 0.\n",
    "        img_tensor[img_tensor>=0.5] = 1.\n",
    "\n",
    "        # Compute signed distances with distance transform\n",
    "        img_tensor = img_tensor.numpy()\n",
    "\n",
    "        neg_distances = scipy.ndimage.morphology.distance_transform_edt(img_tensor)\n",
    "        sd_img = img_tensor - 1.\n",
    "        sd_img = sd_img.astype(np.uint8)\n",
    "        signed_distances = scipy.ndimage.morphology.distance_transform_edt(sd_img) - neg_distances\n",
    "        signed_distances /= float(img_tensor.shape[1])\n",
    "        signed_distances = torch.Tensor(signed_distances)\n",
    "\n",
    "        return signed_distances, torch.Tensor(img_tensor)\n",
    "\n",
    "\n",
    "class MNISTSDFDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, split, size=(256,256)):\n",
    "        self.transform = transforms.Compose([\n",
    "            transforms.Resize(size),\n",
    "            transforms.ToTensor(),\n",
    "            SignedDistanceTransform(),\n",
    "        ])\n",
    "        self.img_dataset = torchvision.datasets.MNIST('./data/MNIST', train=True if split == 'train' else False,\n",
    "                                                download=True)\n",
    "        self.meshgrid = get_mgrid(size[0])\n",
    "        self.im_size = size\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.img_dataset)\n",
    "\n",
    "    def __getitem__(self, item):\n",
    "        img, digit_class = self.img_dataset[item]\n",
    "\n",
    "        signed_distance_img, binary_image = self.transform(img)\n",
    "        \n",
    "        coord_values = self.meshgrid.reshape(-1, 2)\n",
    "        signed_distance_values = signed_distance_img.reshape((-1, 1))\n",
    "        \n",
    "        indices = torch.randperm(coord_values.shape[0])\n",
    "        support_indices = indices[:indices.shape[0]//2]\n",
    "        query_indices = indices[indices.shape[0]//2:]\n",
    "\n",
    "        meta_dict = {'context': {'x':coord_values[support_indices], 'y':signed_distance_values[support_indices]}, \n",
    "                     'query': {'x':coord_values[query_indices], 'y':signed_distance_values[query_indices]}, \n",
    "                     'all': {'x':coord_values, 'y':signed_distance_values}}\n",
    "\n",
    "        return meta_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Q9UUoW_u3V15"
   },
   "source": [
    "Next, we instantiate the dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 437,
     "referenced_widgets": [
      "2b10dbc900014fd3a5a9ab635c6141b8",
      "e41992dfd20a4d9a881efbc938fb8900",
      "ec8ac746cd3a4f9fa8d481da202fa448",
      "fbde554f10584df1944d4d8ccf6a1c0e",
      "91c1f81ea5ef44e4bf7563572bb483db",
      "d739fea08dd64799ae2bc4ec91b8b7ac",
      "58f98fd76dee4e31b72b321ff1af30dd",
      "ea2a8b0c3376426a8798fb7c1e5865e5",
      "d9c9a9be1f1c475bb48b5e6b70f0daaf",
      "98e1375a3e6949e0b5185f82f0036522",
      "b5627888d021415eb4a5961371ad6637",
      "f58247fc00334ffd843487eda6b8aeed",
      "a906a682664e4d53bc827145c851299f",
      "bed4adefda3d45a2bcdce73b40544b17",
      "94a37cbe2d95471dab75148f9eef765c",
      "3fe14aca1a4d44689aa49a7c94566308",
      "cf15b3c559144bdca0661e4c0c181f5b",
      "3781162e8b9c443f8f76ab6fd1bfd017",
      "21163d18f6dd4cec9dc7155ba0d11ca8",
      "0a6b0f3572fe47dc8aaa62bceceb43a6",
      "7d5b5d27bb5e47ba81f33a16f8b30d34",
      "4160c4eda2c94b4ebe7138d1ec271080",
      "8fa8ad4ce6f4474ca8c32999782f2a52",
      "346874e0d31b4312a9ddddcb20fc7c40",
      "0521bcc0b4c5487eb5e600f8bcbed183",
      "fd1f7dc1a92b4728adddb962a52e3d88",
      "a7e799006b0942469dba2c02376cf023",
      "7049e2361a5f4d9c836d97d30ade66da",
      "eccd50d6da84409099dcc84ad6f33431",
      "0fc8bb206a63471d9ce4d774f3079e67",
      "163efd5d3ac74fb6b8cc166b83693c9a",
      "6377c0dc419b4e819afd5b1ad518bdd9"
     ]
    },
    "id": "Zwl8Jlv6Zobh",
    "outputId": "d4ba83be-117f-4977-9dbd-663ba38f0df8"
   },
   "outputs": [],
   "source": [
    "train_dataset = MNISTSDFDataset('train', size=(64, 64))\n",
    "val_dataset = MNISTSDFDataset('val', size=(64, 64))\n",
    "\n",
    "train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=16)\n",
    "val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=16)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nBiUBJ8m3a-0"
   },
   "source": [
    "The losses & a helper function for plotting:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uBUvQtDtZobi"
   },
   "outputs": [],
   "source": [
    "def sdf_loss(predictions, gt):\n",
    "    return ((predictions - gt)**2).mean()\n",
    "\n",
    "\n",
    "def inner_maml_sdf_loss(predictions, gt):\n",
    "    return ((predictions - gt)**2).sum(0).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wxruv24s3sYQ"
   },
   "source": [
    "Next, the initialization scheme proposed by Atzmon et al. 2020:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "onPz1Y1q3rh3"
   },
   "outputs": [],
   "source": [
    "def sal_init(m):\n",
    "    if type(m) == BatchLinear or nn.Linear:\n",
    "        if hasattr(m, 'weight'):\n",
    "            std = np.sqrt(2) / np.sqrt(_calculate_correct_fan(m.weight, 'fan_out'))\n",
    "\n",
    "            with torch.no_grad():\n",
    "                m.weight.normal_(0., std)\n",
    "        if hasattr(m, 'bias'):\n",
    "            m.bias.data.fill_(0.0)\n",
    "\n",
    "\n",
    "def sal_init_last_layer(m):\n",
    "    if hasattr(m, 'weight'):\n",
    "        val = np.sqrt(np.pi) / np.sqrt(_calculate_correct_fan(m.weight, 'fan_in'))\n",
    "        with torch.no_grad():\n",
    "            m.weight.fill_(val)\n",
    "    if hasattr(m, 'bias'):\n",
    "        m.bias.data.fill_(0.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6JK_S5I0Zobj"
   },
   "source": [
    "For this task, we use a simple model with two hidden layers of 256 hidden units. We use the original MAML algorithm with 3 steps and a single learnable LR initialized to 1e-1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "SncWafvNZobj",
    "outputId": "1bfff6cf-128c-4863-cb72-fdc6c0bddad1"
   },
   "outputs": [],
   "source": [
    "hypo_module = MetaFC(in_features=2, out_features=1, \n",
    "                     num_hidden_layers=2, hidden_features=256, \n",
    "                     outermost_linear=True)\n",
    "hypo_module.apply(sal_init)\n",
    "hypo_module.net[-1].apply(sal_init_last_layer)\n",
    "model = MAML(num_meta_steps=3, hypo_module=hypo_module, loss=inner_maml_sdf_loss, init_lr=1e-5, \n",
    "             lr_type='global').cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_DMLuJbF4O23"
   },
   "source": [
    "Now, we're ready for training!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 514
    },
    "id": "pavIvF3XZobk",
    "outputId": "e48a385c-56fd-4982-cc65-ff34e67ea664"
   },
   "outputs": [],
   "source": [
    "optim = torch.optim.Adam(lr=1e-4, params=model.parameters())\n",
    "\n",
    "train_losses = []\n",
    "val_losses = []\n",
    "\n",
    "for epoch in range(3):\n",
    "    for step, meta_batch in enumerate(train_dataloader):\n",
    "        model.train()        \n",
    "        meta_batch = dict_to_gpu(meta_batch)\n",
    "\n",
    "        # Adapt model using context examples\n",
    "        fast_params, _ = model.generate_params(meta_batch['context'])\n",
    "        \n",
    "        # Use the adapted examples to make predictions on query\n",
    "        pred_sd = model.forward_with_params(meta_batch['query']['x'], fast_params)\n",
    "        \n",
    "        # Calculate loss on query examples\n",
    "        loss = sdf_loss(pred_sd, meta_batch['query']['y'].cuda())\n",
    "        train_losses.append(loss.item())\n",
    "\n",
    "        optim.zero_grad()\n",
    "        loss.backward()\n",
    "        optim.step()\n",
    "\n",
    "        if step % 100 == 0:\n",
    "            # Assemble validation input\n",
    "            meta_batch['context']['x'] = meta_batch['query']['x'] = meta_batch['all']['x']\n",
    "            meta_batch['context']['y'] = meta_batch['query']['y'] = meta_batch['all']['y']\n",
    "\n",
    "            with torch.no_grad():\n",
    "                model_output = model(meta_batch)\n",
    "\n",
    "            print(\"Step %d, Total loss %0.6f\" % (step, loss))\n",
    "            fig, axes = plt.subplots(1,5, figsize=(30,6))\n",
    "            ax_titles = ['Learned Initialization', 'Inner step 1 output', \n",
    "                        'Inner step 2 output', 'Inner step 3 output', \n",
    "                        'Ground Truth']\n",
    "            for i, inner_step_out in enumerate(model_output['intermed_predictions']):\n",
    "                plot_sample_image(inner_step_out, ax=axes[i])\n",
    "                axes[i].set_axis_off()\n",
    "                axes[i].set_title(ax_titles[i], fontsize=25)\n",
    "            plot_sample_image(model_output['model_out'], ax=axes[-2])\n",
    "            axes[-2].set_axis_off()\n",
    "            axes[-2].set_title(ax_titles[-2], fontsize=25)\n",
    "\n",
    "            plot_sample_image(meta_batch['query']['y'], ax=axes[-1])\n",
    "            axes[-1].set_axis_off()\n",
    "            axes[-1].set_title(ax_titles[-1], fontsize=25)\n",
    "            plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "with torch.no_grad():\n",
    "    model.eval()\n",
    "    for step, meta_batch in enumerate(val_dataloader):\n",
    "        # Instead of explicitly calling generate_params and forward_with_params,\n",
    "        # we can pass the meta_batch dictionary to the model's forward method\n",
    "        pred_sd = model(meta_batch)\n",
    "        val_loss = sdf_loss(pred_sd['model_out'], meta_batch['query']['y'].cuda())\n",
    "        val_losses.append(val_loss.item())\n",
    "        \n",
    "        if step % 1000 == 0:\n",
    "            pred_image = model.forward_with_params(meta_batch['all']['x'].cuda(), fast_params)\n",
    "            print(f\"Val Image -- Epoch: {epoch} \\t step: {step} \\t loss: {loss.item()}\")\n",
    "            plt.imshow(lin2img(pred_image).cpu().numpy()[0][0])\n",
    "            plt.show()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "MetaSDF.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0521bcc0b4c5487eb5e600f8bcbed183": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_a7e799006b0942469dba2c02376cf023",
       "IPY_MODEL_7049e2361a5f4d9c836d97d30ade66da"
      ],
      "layout": "IPY_MODEL_fd1f7dc1a92b4728adddb962a52e3d88"
     }
    },
    "0a6b0f3572fe47dc8aaa62bceceb43a6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_346874e0d31b4312a9ddddcb20fc7c40",
      "placeholder": "​",
      "style": "IPY_MODEL_8fa8ad4ce6f4474ca8c32999782f2a52",
      "value": " 1654784/? [00:00&lt;00:00, 2289690.28it/s]"
     }
    },
    "0fc8bb206a63471d9ce4d774f3079e67": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "163efd5d3ac74fb6b8cc166b83693c9a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "21163d18f6dd4cec9dc7155ba0d11ca8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4160c4eda2c94b4ebe7138d1ec271080",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_7d5b5d27bb5e47ba81f33a16f8b30d34",
      "value": 1
     }
    },
    "24ac3163a23949079bef466ca70e4bec": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2b10dbc900014fd3a5a9ab635c6141b8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_ec8ac746cd3a4f9fa8d481da202fa448",
       "IPY_MODEL_fbde554f10584df1944d4d8ccf6a1c0e"
      ],
      "layout": "IPY_MODEL_e41992dfd20a4d9a881efbc938fb8900"
     }
    },
    "346874e0d31b4312a9ddddcb20fc7c40": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3781162e8b9c443f8f76ab6fd1bfd017": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3fe14aca1a4d44689aa49a7c94566308": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4098608467004bec96d3b6708c5e0ab0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_5d85b038e8c74f59a10ddec2cf7e7f8d",
       "IPY_MODEL_484a8d36c5174d619cf4492d5c2fc1c3"
      ],
      "layout": "IPY_MODEL_6af97e126e0f401da67392f76628967a"
     }
    },
    "4160c4eda2c94b4ebe7138d1ec271080": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "484a8d36c5174d619cf4492d5c2fc1c3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_24ac3163a23949079bef466ca70e4bec",
      "placeholder": "​",
      "style": "IPY_MODEL_fe8901d5c9484ae59f5d0ba7e851d6b4",
      "value": " 170500096/? [00:07&lt;00:00, 22140814.86it/s]"
     }
    },
    "58f98fd76dee4e31b72b321ff1af30dd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "5d85b038e8c74f59a10ddec2cf7e7f8d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_814d88b570274efc871a7fcff975ae11",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_e1534d9189f84e56bff73e88ecc5913a",
      "value": 1
     }
    },
    "6377c0dc419b4e819afd5b1ad518bdd9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6af97e126e0f401da67392f76628967a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7049e2361a5f4d9c836d97d30ade66da": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_6377c0dc419b4e819afd5b1ad518bdd9",
      "placeholder": "​",
      "style": "IPY_MODEL_163efd5d3ac74fb6b8cc166b83693c9a",
      "value": " 0/4542 [00:00&lt;?, ?it/s]"
     }
    },
    "7d5b5d27bb5e47ba81f33a16f8b30d34": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "814d88b570274efc871a7fcff975ae11": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8fa8ad4ce6f4474ca8c32999782f2a52": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "91c1f81ea5ef44e4bf7563572bb483db": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "94a37cbe2d95471dab75148f9eef765c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "98e1375a3e6949e0b5185f82f0036522": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a7e799006b0942469dba2c02376cf023": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "  0%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0fc8bb206a63471d9ce4d774f3079e67",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_eccd50d6da84409099dcc84ad6f33431",
      "value": 0
     }
    },
    "a906a682664e4d53bc827145c851299f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "b5627888d021415eb4a5961371ad6637": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_bed4adefda3d45a2bcdce73b40544b17",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_a906a682664e4d53bc827145c851299f",
      "value": 1
     }
    },
    "bed4adefda3d45a2bcdce73b40544b17": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cf15b3c559144bdca0661e4c0c181f5b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_21163d18f6dd4cec9dc7155ba0d11ca8",
       "IPY_MODEL_0a6b0f3572fe47dc8aaa62bceceb43a6"
      ],
      "layout": "IPY_MODEL_3781162e8b9c443f8f76ab6fd1bfd017"
     }
    },
    "d739fea08dd64799ae2bc4ec91b8b7ac": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d9c9a9be1f1c475bb48b5e6b70f0daaf": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_b5627888d021415eb4a5961371ad6637",
       "IPY_MODEL_f58247fc00334ffd843487eda6b8aeed"
      ],
      "layout": "IPY_MODEL_98e1375a3e6949e0b5185f82f0036522"
     }
    },
    "e1534d9189f84e56bff73e88ecc5913a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "e41992dfd20a4d9a881efbc938fb8900": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ea2a8b0c3376426a8798fb7c1e5865e5": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ec8ac746cd3a4f9fa8d481da202fa448": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d739fea08dd64799ae2bc4ec91b8b7ac",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_91c1f81ea5ef44e4bf7563572bb483db",
      "value": 1
     }
    },
    "eccd50d6da84409099dcc84ad6f33431": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "f58247fc00334ffd843487eda6b8aeed": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3fe14aca1a4d44689aa49a7c94566308",
      "placeholder": "​",
      "style": "IPY_MODEL_94a37cbe2d95471dab75148f9eef765c",
      "value": " 32768/? [00:00&lt;00:00, 32981.97it/s]"
     }
    },
    "fbde554f10584df1944d4d8ccf6a1c0e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ea2a8b0c3376426a8798fb7c1e5865e5",
      "placeholder": "​",
      "style": "IPY_MODEL_58f98fd76dee4e31b72b321ff1af30dd",
      "value": " 9920512/? [00:02&lt;00:00, 4081994.06it/s]"
     }
    },
    "fd1f7dc1a92b4728adddb962a52e3d88": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "fe8901d5c9484ae59f5d0ba7e851d6b4": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}