small_from_wiki.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Generic-ByT5-pretraining.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MjWEQN6Z--FR"
      },
      "source": [
        "# ByT5 pre-training (generic)\n",
        "\n",
        "Use TPU and High-RAM instance, create a GCS bucket"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_frSR4et-8Ua"
      },
      "source": [
        "## Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X9L9cVEN6QR5"
      },
      "source": [
        "%%capture\n",
        "! pip install t5 sentencepiece apache_beam --upgrade"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1E3JfHHp6aK2",
        "outputId": "e6ef7435-0639-4227-c5bb-53251e3e1c03"
      },
      "source": [
        "! git clone https://github.com/google-research/byt5 byt5-repo\n",
        "! git clone https://github.com/google-research/multilingual-t5\n",
        "! git clone https://github.com/google-research/text-to-text-transfer-transformer"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'byt5-repo'...\n",
            "remote: Enumerating objects: 58, done.\u001b[K\n",
            "remote: Counting objects: 100% (58/58), done.\u001b[K\n",
            "remote: Compressing objects: 100% (35/35), done.\u001b[K\n",
            "remote: Total 58 (delta 26), reused 54 (delta 22), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (58/58), done.\n",
            "Cloning into 'multilingual-t5'...\n",
            "remote: Enumerating objects: 246, done.\u001b[K\n",
            "remote: Counting objects: 100% (69/69), done.\u001b[K\n",
            "remote: Compressing objects: 100% (46/46), done.\u001b[K\n",
            "remote: Total 246 (delta 32), reused 32 (delta 23), pack-reused 177\u001b[K\n",
            "Receiving objects: 100% (246/246), 63.44 KiB | 2.64 MiB/s, done.\n",
            "Resolving deltas: 100% (144/144), done.\n",
            "Cloning into 'text-to-text-transfer-transformer'...\n",
            "remote: Enumerating objects: 3314, done.\u001b[K\n",
            "remote: Counting objects: 100% (259/259), done.\u001b[K\n",
            "remote: Compressing objects: 100% (113/113), done.\u001b[K\n",
            "remote: Total 3314 (delta 154), reused 200 (delta 145), pack-reused 3055\u001b[K\n",
            "Receiving objects: 100% (3314/3314), 5.18 MiB | 15.45 MiB/s, done.\n",
            "Resolving deltas: 100% (2413/2413), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YnxEbRvq_BeC"
      },
      "source": [
        "### Verify"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aOEsb0dB6dL3"
      },
      "source": [
        "from t5.models import mesh_transformer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eplSBJrY77sW"
      },
      "source": [
        "from seqio.dataset_providers import MixtureRegistry, TaskRegistry"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mk6SkUcE_Ggf",
        "outputId": "1439ccf4-b115-4646-b549-b1a52cc86fd1"
      },
      "source": [
        "! python -c \"import t5; print(t5.data.MixtureRegistry.names())\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-07-06 13:35:52.793782: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n",
            "dict_keys([])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P0Ehz6Fv_D3i"
      },
      "source": [
        "## Reorder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rqTvB7sY-tfo"
      },
      "source": [
        "! mv multilingual-t5/multilingual_t5 ./\n",
        "! mv byt5-repo/byt5 ./"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nXj_BJjcAcvs"
      },
      "source": [
        "! mkdir models\n",
        "! cp byt5/gin/models/* models/\n",
        "! cp text-to-text-transfer-transformer/t5/models/gin/models/* models/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "al47b1-y_IwG"
      },
      "source": [
        "## TPU Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ZsOrQ-ARKJ-"
      },
      "source": [
        "# copy this URL into the TPU sections below\n",
        "import os\n",
        "os.environ['COLAB_TPU_ADDR']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TPNvQQ6F-BWA"
      },
      "source": [
        "# in byt5/tasks.py, line 53, check WIKI_LANGS array\n",
        "# you might need to add your lang (ex: \"dv\")\n",
        "# tfds has the other languages; it just wasn't included in the repo here\n",
        "\n",
        "# saves model at step 0, 5100, 10100 (?)\n",
        "\n",
        "! python -m t5.models.mesh_transformer_main \\\n",
        "  --gin_file=\"./models/byt5.small.gin\" \\ # could be byt5.large or other .gin templates\n",
        "  --gin_param=\"MIXTURE_NAME = 'byt5_wiki.LANG'\" \\ # update me\n",
        "  --gin_param=\"mesh_train_dataset_fn.mixture_or_task_name = 'byt5_wiki.LANG'\" \\ # update me\n",
        "  --gin_param=\"utils.run.sequence_length = {'inputs': 1024, 'targets': 189}\" \\\n",
        "  --gin_param=\"utils.run.batch_size = ('tokens_per_batch', 262144)\" \\ # 1/4 of readme\n",
        "  --gin_param=\"run.train_steps = 100000\" \\ # 1/10 of readme\n",
        "  --gin_param=\"utils.tpu_mesh_shape.tpu_topology = 'v3-8'\" \\ # on CoLab you only get 8 cores\n",
        "  --gin_param=\"run.train_dataset_fn = @t5.models.mesh_transformer.mesh_train_dataset_fn\" \\\n",
        "  --module_import=\"byt5.tasks\" \\\n",
        "  --tpu=\"grpc://TPU\" \\ # TPU address\n",
        "  --gin_param=\"utils.tpu_mesh_shape.model_parallelism = 1\" \\\n",
        "  --model_dir=\"gs://BUCKET/byt5_model\" \\ # your GCS bucket name\n",
        "  --gcp_project=\"GCP\" # your GCP project\n",
        "\n",
        "  #--t5_tfds_data_dir=\"${BUCKET}/t5-tfds\" \\\n",
        "    # --eval_mode=\"perplexity_eval\" \\\n",
        "#  --eval_gin_param=\"mesh_eval_dataset_fn.num_eval_examples = 10000\" \\\n",
        "  #--tpu_zone=\"${ZONE}\" \\\n",
        "#   --gin_param=\"utils.run.learning_rate_schedule=@learning_rate_schedules.rsqrt_no_ramp_down\" \\\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OvSDo2jJge8P"
      },
      "source": [
        "## Convert checkpoint to HF / PyTorch model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fuG6SG-0gify"
      },
      "source": [
        "%%capture\n",
        "! pip install transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XrPASnRQkO7L"
      },
      "source": [
        "! cp ./drive/MyDrive/mlin/dvcorpus/dv-t5/checkpoint ./drive/MyDrive/mlin/dvcorpus/dv-t5/model.ckpt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nXb4wtm8gebP",
        "outputId": "a50bc029-d409-40c0-9da5-aca9ef22782e"
      },
      "source": [
        "! transformers-cli convert --model_type t5 \\\n",
        "  --tf_checkpoint ./drive/MyDrive/mlin/dvcorpus/dv-t5/ \\\n",
        "  --config ./drive/MyDrive/mlin/dvcorpus/dv-t5/config.json \\\n",
        "  --pytorch_dump_output ./drive/MyDrive/mlin/dvcorpus/dv-t5/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-07-06 20:45:17.975878: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n",
            "Building PyTorch model from configuration: T5Config {\n",
            "  \"architectures\": [\n",
            "    \"T5ForConditionalGeneration\"\n",
            "  ],\n",
            "  \"d_ff\": 3584,\n",
            "  \"d_kv\": 64,\n",
            "  \"d_model\": 1472,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"dropout_rate\": 0.1,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"feed_forward_proj\": \"gated-gelu\",\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_factor\": 1.0,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"layer_norm_epsilon\": 1e-06,\n",
            "  \"model_type\": \"t5\",\n",
            "  \"num_decoder_layers\": 4,\n",
            "  \"num_heads\": 6,\n",
            "  \"num_layers\": 12,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"relative_attention_num_buckets\": 32,\n",
            "  \"tie_word_embeddings\": false,\n",
            "  \"tokenizer_class\": \"ByT5Tokenizer\",\n",
            "  \"transformers_version\": \"4.8.2\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 384\n",
            "}\n",
            "\n",
            "Converting TensorFlow checkpoint from /content/drive/MyDrive/mlin/dvcorpus/dv-t5\n",
            "Loading TF weight decoder/block_000/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight decoder/block_000/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_000/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight decoder/block_000/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_000/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight decoder/block_000/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_000/layer_000/SelfAttention/relative_attention_bias with shape [6, 32]\n",
            "Loading TF weight decoder/block_000/layer_000/SelfAttention/relative_attention_bias_slot_v with shape [6, 32]\n",
            "Loading TF weight decoder/block_000/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight decoder/block_000/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_000/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_001/EncDecAttention/k with shape [1472, 384]\n",
            "Loading TF weight decoder/block_000/layer_001/EncDecAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_001/EncDecAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_000/layer_001/EncDecAttention/o with shape [384, 1472]\n",
            "Loading TF weight decoder/block_000/layer_001/EncDecAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_001/EncDecAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_000/layer_001/EncDecAttention/q with shape [1472, 384]\n",
            "Loading TF weight decoder/block_000/layer_001/EncDecAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_001/EncDecAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_000/layer_001/EncDecAttention/v with shape [1472, 384]\n",
            "Loading TF weight decoder/block_000/layer_001/EncDecAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_001/EncDecAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_000/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_002/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight decoder/block_000/layer_002/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight decoder/block_000/layer_002/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_002/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight decoder/block_000/layer_002/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight decoder/block_000/layer_002/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_002/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight decoder/block_000/layer_002/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight decoder/block_000/layer_002/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_002/rms_norm/scale with shape [1472]\n",
            "Loading TF weight decoder/block_000/layer_002/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight decoder/block_001/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_001/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight decoder/block_001/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_001/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight decoder/block_001/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_001/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight decoder/block_001/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_001/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_001/EncDecAttention/k with shape [1472, 384]\n",
            "Loading TF weight decoder/block_001/layer_001/EncDecAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_001/EncDecAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_001/layer_001/EncDecAttention/o with shape [384, 1472]\n",
            "Loading TF weight decoder/block_001/layer_001/EncDecAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_001/EncDecAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_001/layer_001/EncDecAttention/q with shape [1472, 384]\n",
            "Loading TF weight decoder/block_001/layer_001/EncDecAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_001/EncDecAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_001/layer_001/EncDecAttention/v with shape [1472, 384]\n",
            "Loading TF weight decoder/block_001/layer_001/EncDecAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_001/EncDecAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_001/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_002/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight decoder/block_001/layer_002/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight decoder/block_001/layer_002/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_002/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight decoder/block_001/layer_002/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight decoder/block_001/layer_002/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_002/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight decoder/block_001/layer_002/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight decoder/block_001/layer_002/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_002/rms_norm/scale with shape [1472]\n",
            "Loading TF weight decoder/block_001/layer_002/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight decoder/block_002/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_002/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight decoder/block_002/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_002/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight decoder/block_002/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_002/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight decoder/block_002/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_002/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_001/EncDecAttention/k with shape [1472, 384]\n",
            "Loading TF weight decoder/block_002/layer_001/EncDecAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_001/EncDecAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_002/layer_001/EncDecAttention/o with shape [384, 1472]\n",
            "Loading TF weight decoder/block_002/layer_001/EncDecAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_001/EncDecAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_002/layer_001/EncDecAttention/q with shape [1472, 384]\n",
            "Loading TF weight decoder/block_002/layer_001/EncDecAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_001/EncDecAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_002/layer_001/EncDecAttention/v with shape [1472, 384]\n",
            "Loading TF weight decoder/block_002/layer_001/EncDecAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_001/EncDecAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_002/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_002/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight decoder/block_002/layer_002/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight decoder/block_002/layer_002/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_002/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight decoder/block_002/layer_002/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight decoder/block_002/layer_002/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_002/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight decoder/block_002/layer_002/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight decoder/block_002/layer_002/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_002/rms_norm/scale with shape [1472]\n",
            "Loading TF weight decoder/block_002/layer_002/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight decoder/block_003/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_003/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight decoder/block_003/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_003/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight decoder/block_003/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_003/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight decoder/block_003/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_003/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_001/EncDecAttention/k with shape [1472, 384]\n",
            "Loading TF weight decoder/block_003/layer_001/EncDecAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_001/EncDecAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_003/layer_001/EncDecAttention/o with shape [384, 1472]\n",
            "Loading TF weight decoder/block_003/layer_001/EncDecAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_001/EncDecAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_003/layer_001/EncDecAttention/q with shape [1472, 384]\n",
            "Loading TF weight decoder/block_003/layer_001/EncDecAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_001/EncDecAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_003/layer_001/EncDecAttention/v with shape [1472, 384]\n",
            "Loading TF weight decoder/block_003/layer_001/EncDecAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_001/EncDecAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight decoder/block_003/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_002/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight decoder/block_003/layer_002/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight decoder/block_003/layer_002/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_002/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight decoder/block_003/layer_002/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight decoder/block_003/layer_002/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_002/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight decoder/block_003/layer_002/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight decoder/block_003/layer_002/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_002/rms_norm/scale with shape [1472]\n",
            "Loading TF weight decoder/block_003/layer_002/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight decoder/logits/kernel with shape [1472, 384]\n",
            "Loading TF weight decoder/logits/kernel_slot_vc with shape [1472]\n",
            "Loading TF weight decoder/logits/kernel_slot_vr with shape [384]\n",
            "Loading TF weight decoder/rms_norm/scale with shape [1472]\n",
            "Loading TF weight decoder/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_000/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight encoder/block_000/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_000/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_000/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight encoder/block_000/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_000/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_000/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight encoder/block_000/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_000/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_000/layer_000/SelfAttention/relative_attention_bias with shape [6, 32]\n",
            "Loading TF weight encoder/block_000/layer_000/SelfAttention/relative_attention_bias_slot_v with shape [6, 32]\n",
            "Loading TF weight encoder/block_000/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight encoder/block_000/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_000/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_000/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_000/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_000/layer_001/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_000/layer_001/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_000/layer_001/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_000/layer_001/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_000/layer_001/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_000/layer_001/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_000/layer_001/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight encoder/block_000/layer_001/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_000/layer_001/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_000/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_000/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_001/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight encoder/block_001/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_001/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_001/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight encoder/block_001/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_001/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_001/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight encoder/block_001/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_001/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_001/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight encoder/block_001/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_001/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_001/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_001/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_001/layer_001/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_001/layer_001/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_001/layer_001/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_001/layer_001/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_001/layer_001/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_001/layer_001/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_001/layer_001/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight encoder/block_001/layer_001/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_001/layer_001/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_001/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_001/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_002/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight encoder/block_002/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_002/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_002/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight encoder/block_002/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_002/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_002/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight encoder/block_002/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_002/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_002/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight encoder/block_002/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_002/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_002/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_002/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_002/layer_001/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_002/layer_001/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_002/layer_001/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_002/layer_001/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_002/layer_001/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_002/layer_001/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_002/layer_001/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight encoder/block_002/layer_001/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_002/layer_001/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_002/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_002/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_003/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight encoder/block_003/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_003/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_003/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight encoder/block_003/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_003/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_003/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight encoder/block_003/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_003/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_003/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight encoder/block_003/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_003/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_003/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_003/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_003/layer_001/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_003/layer_001/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_003/layer_001/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_003/layer_001/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_003/layer_001/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_003/layer_001/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_003/layer_001/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight encoder/block_003/layer_001/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_003/layer_001/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_003/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_003/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_004/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight encoder/block_004/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_004/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_004/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight encoder/block_004/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_004/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_004/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight encoder/block_004/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_004/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_004/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight encoder/block_004/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_004/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_004/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_004/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_004/layer_001/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_004/layer_001/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_004/layer_001/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_004/layer_001/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_004/layer_001/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_004/layer_001/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_004/layer_001/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight encoder/block_004/layer_001/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_004/layer_001/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_004/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_004/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_005/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight encoder/block_005/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_005/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_005/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight encoder/block_005/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_005/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_005/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight encoder/block_005/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_005/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_005/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight encoder/block_005/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_005/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_005/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_005/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_005/layer_001/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_005/layer_001/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_005/layer_001/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_005/layer_001/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_005/layer_001/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_005/layer_001/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_005/layer_001/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight encoder/block_005/layer_001/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_005/layer_001/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_005/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_005/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_006/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight encoder/block_006/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_006/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_006/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight encoder/block_006/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_006/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_006/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight encoder/block_006/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_006/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_006/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight encoder/block_006/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_006/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_006/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_006/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_006/layer_001/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_006/layer_001/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_006/layer_001/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_006/layer_001/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_006/layer_001/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_006/layer_001/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_006/layer_001/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight encoder/block_006/layer_001/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_006/layer_001/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_006/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_006/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_007/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight encoder/block_007/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_007/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_007/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight encoder/block_007/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_007/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_007/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight encoder/block_007/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_007/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_007/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight encoder/block_007/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_007/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_007/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_007/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_007/layer_001/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_007/layer_001/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_007/layer_001/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_007/layer_001/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_007/layer_001/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_007/layer_001/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_007/layer_001/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight encoder/block_007/layer_001/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_007/layer_001/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_007/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_007/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_008/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight encoder/block_008/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_008/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_008/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight encoder/block_008/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_008/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_008/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight encoder/block_008/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_008/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_008/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight encoder/block_008/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_008/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_008/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_008/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_008/layer_001/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_008/layer_001/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_008/layer_001/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_008/layer_001/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_008/layer_001/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_008/layer_001/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_008/layer_001/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight encoder/block_008/layer_001/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_008/layer_001/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_008/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_008/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_009/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight encoder/block_009/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_009/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_009/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight encoder/block_009/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_009/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_009/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight encoder/block_009/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_009/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_009/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight encoder/block_009/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_009/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_009/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_009/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_009/layer_001/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_009/layer_001/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_009/layer_001/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_009/layer_001/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_009/layer_001/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_009/layer_001/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_009/layer_001/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight encoder/block_009/layer_001/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_009/layer_001/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_009/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_009/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_010/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight encoder/block_010/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_010/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_010/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight encoder/block_010/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_010/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_010/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight encoder/block_010/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_010/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_010/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight encoder/block_010/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_010/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_010/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_010/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_010/layer_001/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_010/layer_001/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_010/layer_001/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_010/layer_001/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_010/layer_001/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_010/layer_001/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_010/layer_001/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight encoder/block_010/layer_001/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_010/layer_001/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_010/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_010/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_011/layer_000/SelfAttention/k with shape [1472, 384]\n",
            "Loading TF weight encoder/block_011/layer_000/SelfAttention/k_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_011/layer_000/SelfAttention/k_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_011/layer_000/SelfAttention/o with shape [384, 1472]\n",
            "Loading TF weight encoder/block_011/layer_000/SelfAttention/o_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_011/layer_000/SelfAttention/o_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_011/layer_000/SelfAttention/q with shape [1472, 384]\n",
            "Loading TF weight encoder/block_011/layer_000/SelfAttention/q_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_011/layer_000/SelfAttention/q_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_011/layer_000/SelfAttention/v with shape [1472, 384]\n",
            "Loading TF weight encoder/block_011/layer_000/SelfAttention/v_slot_vc with shape [1472]\n",
            "Loading TF weight encoder/block_011/layer_000/SelfAttention/v_slot_vr with shape [384]\n",
            "Loading TF weight encoder/block_011/layer_000/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_011/layer_000/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/block_011/layer_001/DenseReluDense/wi_0/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_011/layer_001/DenseReluDense/wi_0/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_011/layer_001/DenseReluDense/wi_0/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_011/layer_001/DenseReluDense/wi_1/kernel with shape [1472, 3584]\n",
            "Loading TF weight encoder/block_011/layer_001/DenseReluDense/wi_1/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_011/layer_001/DenseReluDense/wi_1/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_011/layer_001/DenseReluDense/wo/kernel with shape [3584, 1472]\n",
            "Loading TF weight encoder/block_011/layer_001/DenseReluDense/wo/kernel_slot_vc with shape [3584]\n",
            "Loading TF weight encoder/block_011/layer_001/DenseReluDense/wo/kernel_slot_vr with shape [1472]\n",
            "Loading TF weight encoder/block_011/layer_001/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/block_011/layer_001/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight encoder/rms_norm/scale with shape [1472]\n",
            "Loading TF weight encoder/rms_norm/scale_slot_v with shape [1472]\n",
            "Loading TF weight global_step with shape []\n",
            "Loading TF weight shared/embedding with shape [384, 1472]\n",
            "Loading TF weight shared/embedding_slot_vc with shape [1472]\n",
            "Loading TF weight shared/embedding_slot_vr with shape [384]\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_000', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['decoder', 'block_000', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping decoder/block_000/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping decoder/block_000/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['decoder', 'block_000', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['decoder', 'block_000', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping decoder/block_000/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping decoder/block_000/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_000', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['decoder', 'block_000', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping decoder/block_000/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping decoder/block_000/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (6, 32) for ['decoder', 'block_000', 'layer_000', 'SelfAttention', 'relative_attention_bias']\n",
            "Initialize PyTorch weight ['decoder', 'block_000', 'layer_000', 'SelfAttention', 'relative_attention_bias']\n",
            "Skipping decoder/block_000/layer_000/SelfAttention/relative_attention_bias_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_000', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['decoder', 'block_000', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping decoder/block_000/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping decoder/block_000/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['decoder', 'block_000', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['decoder', 'block_000', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping decoder/block_000/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_000', 'layer_001', 'EncDecAttention', 'k']\n",
            "Initialize PyTorch weight ['decoder', 'block_000', 'layer_001', 'EncDecAttention', 'k']\n",
            "Skipping decoder/block_000/layer_001/EncDecAttention/k_slot_vc\n",
            "Skipping decoder/block_000/layer_001/EncDecAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['decoder', 'block_000', 'layer_001', 'EncDecAttention', 'o']\n",
            "Initialize PyTorch weight ['decoder', 'block_000', 'layer_001', 'EncDecAttention', 'o']\n",
            "Skipping decoder/block_000/layer_001/EncDecAttention/o_slot_vc\n",
            "Skipping decoder/block_000/layer_001/EncDecAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_000', 'layer_001', 'EncDecAttention', 'q']\n",
            "Initialize PyTorch weight ['decoder', 'block_000', 'layer_001', 'EncDecAttention', 'q']\n",
            "Skipping decoder/block_000/layer_001/EncDecAttention/q_slot_vc\n",
            "Skipping decoder/block_000/layer_001/EncDecAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_000', 'layer_001', 'EncDecAttention', 'v']\n",
            "Initialize PyTorch weight ['decoder', 'block_000', 'layer_001', 'EncDecAttention', 'v']\n",
            "Skipping decoder/block_000/layer_001/EncDecAttention/v_slot_vc\n",
            "Skipping decoder/block_000/layer_001/EncDecAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['decoder', 'block_000', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['decoder', 'block_000', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping decoder/block_000/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['decoder', 'block_000', 'layer_002', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['decoder', 'block_000', 'layer_002', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping decoder/block_000/layer_002/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping decoder/block_000/layer_002/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['decoder', 'block_000', 'layer_002', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['decoder', 'block_000', 'layer_002', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping decoder/block_000/layer_002/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping decoder/block_000/layer_002/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['decoder', 'block_000', 'layer_002', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['decoder', 'block_000', 'layer_002', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping decoder/block_000/layer_002/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping decoder/block_000/layer_002/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['decoder', 'block_000', 'layer_002', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['decoder', 'block_000', 'layer_002', 'rms_norm', 'scale']\n",
            "Skipping decoder/block_000/layer_002/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_001', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['decoder', 'block_001', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping decoder/block_001/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping decoder/block_001/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['decoder', 'block_001', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['decoder', 'block_001', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping decoder/block_001/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping decoder/block_001/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_001', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['decoder', 'block_001', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping decoder/block_001/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping decoder/block_001/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_001', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['decoder', 'block_001', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping decoder/block_001/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping decoder/block_001/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['decoder', 'block_001', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['decoder', 'block_001', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping decoder/block_001/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_001', 'layer_001', 'EncDecAttention', 'k']\n",
            "Initialize PyTorch weight ['decoder', 'block_001', 'layer_001', 'EncDecAttention', 'k']\n",
            "Skipping decoder/block_001/layer_001/EncDecAttention/k_slot_vc\n",
            "Skipping decoder/block_001/layer_001/EncDecAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['decoder', 'block_001', 'layer_001', 'EncDecAttention', 'o']\n",
            "Initialize PyTorch weight ['decoder', 'block_001', 'layer_001', 'EncDecAttention', 'o']\n",
            "Skipping decoder/block_001/layer_001/EncDecAttention/o_slot_vc\n",
            "Skipping decoder/block_001/layer_001/EncDecAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_001', 'layer_001', 'EncDecAttention', 'q']\n",
            "Initialize PyTorch weight ['decoder', 'block_001', 'layer_001', 'EncDecAttention', 'q']\n",
            "Skipping decoder/block_001/layer_001/EncDecAttention/q_slot_vc\n",
            "Skipping decoder/block_001/layer_001/EncDecAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_001', 'layer_001', 'EncDecAttention', 'v']\n",
            "Initialize PyTorch weight ['decoder', 'block_001', 'layer_001', 'EncDecAttention', 'v']\n",
            "Skipping decoder/block_001/layer_001/EncDecAttention/v_slot_vc\n",
            "Skipping decoder/block_001/layer_001/EncDecAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['decoder', 'block_001', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['decoder', 'block_001', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping decoder/block_001/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['decoder', 'block_001', 'layer_002', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['decoder', 'block_001', 'layer_002', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping decoder/block_001/layer_002/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping decoder/block_001/layer_002/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['decoder', 'block_001', 'layer_002', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['decoder', 'block_001', 'layer_002', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping decoder/block_001/layer_002/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping decoder/block_001/layer_002/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['decoder', 'block_001', 'layer_002', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['decoder', 'block_001', 'layer_002', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping decoder/block_001/layer_002/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping decoder/block_001/layer_002/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['decoder', 'block_001', 'layer_002', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['decoder', 'block_001', 'layer_002', 'rms_norm', 'scale']\n",
            "Skipping decoder/block_001/layer_002/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_002', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['decoder', 'block_002', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping decoder/block_002/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping decoder/block_002/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['decoder', 'block_002', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['decoder', 'block_002', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping decoder/block_002/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping decoder/block_002/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_002', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['decoder', 'block_002', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping decoder/block_002/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping decoder/block_002/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_002', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['decoder', 'block_002', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping decoder/block_002/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping decoder/block_002/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['decoder', 'block_002', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['decoder', 'block_002', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping decoder/block_002/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_002', 'layer_001', 'EncDecAttention', 'k']\n",
            "Initialize PyTorch weight ['decoder', 'block_002', 'layer_001', 'EncDecAttention', 'k']\n",
            "Skipping decoder/block_002/layer_001/EncDecAttention/k_slot_vc\n",
            "Skipping decoder/block_002/layer_001/EncDecAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['decoder', 'block_002', 'layer_001', 'EncDecAttention', 'o']\n",
            "Initialize PyTorch weight ['decoder', 'block_002', 'layer_001', 'EncDecAttention', 'o']\n",
            "Skipping decoder/block_002/layer_001/EncDecAttention/o_slot_vc\n",
            "Skipping decoder/block_002/layer_001/EncDecAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_002', 'layer_001', 'EncDecAttention', 'q']\n",
            "Initialize PyTorch weight ['decoder', 'block_002', 'layer_001', 'EncDecAttention', 'q']\n",
            "Skipping decoder/block_002/layer_001/EncDecAttention/q_slot_vc\n",
            "Skipping decoder/block_002/layer_001/EncDecAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_002', 'layer_001', 'EncDecAttention', 'v']\n",
            "Initialize PyTorch weight ['decoder', 'block_002', 'layer_001', 'EncDecAttention', 'v']\n",
            "Skipping decoder/block_002/layer_001/EncDecAttention/v_slot_vc\n",
            "Skipping decoder/block_002/layer_001/EncDecAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['decoder', 'block_002', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['decoder', 'block_002', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping decoder/block_002/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['decoder', 'block_002', 'layer_002', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['decoder', 'block_002', 'layer_002', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping decoder/block_002/layer_002/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping decoder/block_002/layer_002/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['decoder', 'block_002', 'layer_002', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['decoder', 'block_002', 'layer_002', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping decoder/block_002/layer_002/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping decoder/block_002/layer_002/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['decoder', 'block_002', 'layer_002', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['decoder', 'block_002', 'layer_002', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping decoder/block_002/layer_002/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping decoder/block_002/layer_002/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['decoder', 'block_002', 'layer_002', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['decoder', 'block_002', 'layer_002', 'rms_norm', 'scale']\n",
            "Skipping decoder/block_002/layer_002/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_003', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['decoder', 'block_003', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping decoder/block_003/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping decoder/block_003/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['decoder', 'block_003', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['decoder', 'block_003', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping decoder/block_003/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping decoder/block_003/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_003', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['decoder', 'block_003', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping decoder/block_003/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping decoder/block_003/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_003', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['decoder', 'block_003', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping decoder/block_003/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping decoder/block_003/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['decoder', 'block_003', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['decoder', 'block_003', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping decoder/block_003/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_003', 'layer_001', 'EncDecAttention', 'k']\n",
            "Initialize PyTorch weight ['decoder', 'block_003', 'layer_001', 'EncDecAttention', 'k']\n",
            "Skipping decoder/block_003/layer_001/EncDecAttention/k_slot_vc\n",
            "Skipping decoder/block_003/layer_001/EncDecAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['decoder', 'block_003', 'layer_001', 'EncDecAttention', 'o']\n",
            "Initialize PyTorch weight ['decoder', 'block_003', 'layer_001', 'EncDecAttention', 'o']\n",
            "Skipping decoder/block_003/layer_001/EncDecAttention/o_slot_vc\n",
            "Skipping decoder/block_003/layer_001/EncDecAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_003', 'layer_001', 'EncDecAttention', 'q']\n",
            "Initialize PyTorch weight ['decoder', 'block_003', 'layer_001', 'EncDecAttention', 'q']\n",
            "Skipping decoder/block_003/layer_001/EncDecAttention/q_slot_vc\n",
            "Skipping decoder/block_003/layer_001/EncDecAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'block_003', 'layer_001', 'EncDecAttention', 'v']\n",
            "Initialize PyTorch weight ['decoder', 'block_003', 'layer_001', 'EncDecAttention', 'v']\n",
            "Skipping decoder/block_003/layer_001/EncDecAttention/v_slot_vc\n",
            "Skipping decoder/block_003/layer_001/EncDecAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['decoder', 'block_003', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['decoder', 'block_003', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping decoder/block_003/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['decoder', 'block_003', 'layer_002', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['decoder', 'block_003', 'layer_002', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping decoder/block_003/layer_002/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping decoder/block_003/layer_002/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['decoder', 'block_003', 'layer_002', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['decoder', 'block_003', 'layer_002', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping decoder/block_003/layer_002/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping decoder/block_003/layer_002/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['decoder', 'block_003', 'layer_002', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['decoder', 'block_003', 'layer_002', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping decoder/block_003/layer_002/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping decoder/block_003/layer_002/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['decoder', 'block_003', 'layer_002', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['decoder', 'block_003', 'layer_002', 'rms_norm', 'scale']\n",
            "Skipping decoder/block_003/layer_002/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['decoder', 'logits', 'kernel']\n",
            "Initialize PyTorch weight ['decoder', 'logits', 'kernel']\n",
            "Skipping decoder/logits/kernel_slot_vc\n",
            "Skipping decoder/logits/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['decoder', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['decoder', 'rms_norm', 'scale']\n",
            "Skipping decoder/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_000', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['encoder', 'block_000', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping encoder/block_000/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping encoder/block_000/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['encoder', 'block_000', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['encoder', 'block_000', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping encoder/block_000/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping encoder/block_000/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_000', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['encoder', 'block_000', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping encoder/block_000/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping encoder/block_000/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (6, 32) for ['encoder', 'block_000', 'layer_000', 'SelfAttention', 'relative_attention_bias']\n",
            "Initialize PyTorch weight ['encoder', 'block_000', 'layer_000', 'SelfAttention', 'relative_attention_bias']\n",
            "Skipping encoder/block_000/layer_000/SelfAttention/relative_attention_bias_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_000', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['encoder', 'block_000', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping encoder/block_000/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping encoder/block_000/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_000', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_000', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_000/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_000', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_000', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping encoder/block_000/layer_001/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping encoder/block_000/layer_001/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_000', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_000', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping encoder/block_000/layer_001/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping encoder/block_000/layer_001/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['encoder', 'block_000', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_000', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping encoder/block_000/layer_001/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping encoder/block_000/layer_001/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_000', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_000', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_000/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_001', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['encoder', 'block_001', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping encoder/block_001/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping encoder/block_001/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['encoder', 'block_001', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['encoder', 'block_001', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping encoder/block_001/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping encoder/block_001/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_001', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['encoder', 'block_001', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping encoder/block_001/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping encoder/block_001/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_001', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['encoder', 'block_001', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping encoder/block_001/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping encoder/block_001/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_001', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_001', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_001/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_001', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_001', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping encoder/block_001/layer_001/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping encoder/block_001/layer_001/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_001', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_001', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping encoder/block_001/layer_001/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping encoder/block_001/layer_001/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['encoder', 'block_001', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_001', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping encoder/block_001/layer_001/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping encoder/block_001/layer_001/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_001', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_001', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_001/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_002', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['encoder', 'block_002', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping encoder/block_002/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping encoder/block_002/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['encoder', 'block_002', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['encoder', 'block_002', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping encoder/block_002/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping encoder/block_002/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_002', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['encoder', 'block_002', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping encoder/block_002/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping encoder/block_002/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_002', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['encoder', 'block_002', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping encoder/block_002/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping encoder/block_002/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_002', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_002', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_002/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_002', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_002', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping encoder/block_002/layer_001/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping encoder/block_002/layer_001/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_002', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_002', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping encoder/block_002/layer_001/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping encoder/block_002/layer_001/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['encoder', 'block_002', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_002', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping encoder/block_002/layer_001/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping encoder/block_002/layer_001/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_002', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_002', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_002/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_003', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['encoder', 'block_003', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping encoder/block_003/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping encoder/block_003/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['encoder', 'block_003', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['encoder', 'block_003', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping encoder/block_003/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping encoder/block_003/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_003', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['encoder', 'block_003', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping encoder/block_003/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping encoder/block_003/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_003', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['encoder', 'block_003', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping encoder/block_003/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping encoder/block_003/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_003', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_003', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_003/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_003', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_003', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping encoder/block_003/layer_001/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping encoder/block_003/layer_001/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_003', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_003', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping encoder/block_003/layer_001/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping encoder/block_003/layer_001/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['encoder', 'block_003', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_003', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping encoder/block_003/layer_001/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping encoder/block_003/layer_001/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_003', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_003', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_003/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_004', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['encoder', 'block_004', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping encoder/block_004/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping encoder/block_004/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['encoder', 'block_004', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['encoder', 'block_004', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping encoder/block_004/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping encoder/block_004/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_004', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['encoder', 'block_004', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping encoder/block_004/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping encoder/block_004/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_004', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['encoder', 'block_004', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping encoder/block_004/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping encoder/block_004/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_004', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_004', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_004/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_004', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_004', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping encoder/block_004/layer_001/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping encoder/block_004/layer_001/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_004', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_004', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping encoder/block_004/layer_001/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping encoder/block_004/layer_001/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['encoder', 'block_004', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_004', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping encoder/block_004/layer_001/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping encoder/block_004/layer_001/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_004', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_004', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_004/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_005', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['encoder', 'block_005', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping encoder/block_005/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping encoder/block_005/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['encoder', 'block_005', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['encoder', 'block_005', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping encoder/block_005/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping encoder/block_005/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_005', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['encoder', 'block_005', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping encoder/block_005/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping encoder/block_005/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_005', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['encoder', 'block_005', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping encoder/block_005/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping encoder/block_005/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_005', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_005', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_005/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_005', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_005', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping encoder/block_005/layer_001/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping encoder/block_005/layer_001/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_005', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_005', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping encoder/block_005/layer_001/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping encoder/block_005/layer_001/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['encoder', 'block_005', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_005', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping encoder/block_005/layer_001/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping encoder/block_005/layer_001/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_005', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_005', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_005/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_006', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['encoder', 'block_006', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping encoder/block_006/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping encoder/block_006/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['encoder', 'block_006', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['encoder', 'block_006', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping encoder/block_006/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping encoder/block_006/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_006', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['encoder', 'block_006', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping encoder/block_006/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping encoder/block_006/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_006', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['encoder', 'block_006', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping encoder/block_006/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping encoder/block_006/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_006', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_006', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_006/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_006', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_006', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping encoder/block_006/layer_001/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping encoder/block_006/layer_001/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_006', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_006', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping encoder/block_006/layer_001/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping encoder/block_006/layer_001/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['encoder', 'block_006', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_006', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping encoder/block_006/layer_001/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping encoder/block_006/layer_001/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_006', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_006', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_006/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_007', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['encoder', 'block_007', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping encoder/block_007/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping encoder/block_007/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['encoder', 'block_007', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['encoder', 'block_007', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping encoder/block_007/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping encoder/block_007/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_007', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['encoder', 'block_007', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping encoder/block_007/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping encoder/block_007/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_007', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['encoder', 'block_007', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping encoder/block_007/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping encoder/block_007/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_007', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_007', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_007/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_007', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_007', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping encoder/block_007/layer_001/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping encoder/block_007/layer_001/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_007', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_007', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping encoder/block_007/layer_001/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping encoder/block_007/layer_001/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['encoder', 'block_007', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_007', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping encoder/block_007/layer_001/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping encoder/block_007/layer_001/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_007', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_007', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_007/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_008', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['encoder', 'block_008', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping encoder/block_008/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping encoder/block_008/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['encoder', 'block_008', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['encoder', 'block_008', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping encoder/block_008/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping encoder/block_008/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_008', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['encoder', 'block_008', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping encoder/block_008/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping encoder/block_008/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_008', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['encoder', 'block_008', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping encoder/block_008/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping encoder/block_008/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_008', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_008', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_008/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_008', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_008', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping encoder/block_008/layer_001/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping encoder/block_008/layer_001/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_008', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_008', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping encoder/block_008/layer_001/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping encoder/block_008/layer_001/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['encoder', 'block_008', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_008', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping encoder/block_008/layer_001/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping encoder/block_008/layer_001/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_008', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_008', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_008/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_009', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['encoder', 'block_009', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping encoder/block_009/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping encoder/block_009/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['encoder', 'block_009', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['encoder', 'block_009', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping encoder/block_009/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping encoder/block_009/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_009', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['encoder', 'block_009', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping encoder/block_009/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping encoder/block_009/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_009', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['encoder', 'block_009', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping encoder/block_009/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping encoder/block_009/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_009', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_009', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_009/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_009', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_009', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping encoder/block_009/layer_001/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping encoder/block_009/layer_001/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_009', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_009', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping encoder/block_009/layer_001/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping encoder/block_009/layer_001/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['encoder', 'block_009', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_009', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping encoder/block_009/layer_001/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping encoder/block_009/layer_001/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_009', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_009', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_009/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_010', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['encoder', 'block_010', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping encoder/block_010/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping encoder/block_010/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['encoder', 'block_010', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['encoder', 'block_010', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping encoder/block_010/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping encoder/block_010/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_010', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['encoder', 'block_010', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping encoder/block_010/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping encoder/block_010/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_010', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['encoder', 'block_010', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping encoder/block_010/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping encoder/block_010/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_010', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_010', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_010/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_010', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_010', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping encoder/block_010/layer_001/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping encoder/block_010/layer_001/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_010', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_010', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping encoder/block_010/layer_001/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping encoder/block_010/layer_001/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['encoder', 'block_010', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_010', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping encoder/block_010/layer_001/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping encoder/block_010/layer_001/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_010', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_010', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_010/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_011', 'layer_000', 'SelfAttention', 'k']\n",
            "Initialize PyTorch weight ['encoder', 'block_011', 'layer_000', 'SelfAttention', 'k']\n",
            "Skipping encoder/block_011/layer_000/SelfAttention/k_slot_vc\n",
            "Skipping encoder/block_011/layer_000/SelfAttention/k_slot_vr\n",
            "Transposing numpy weight of shape (384, 1472) for ['encoder', 'block_011', 'layer_000', 'SelfAttention', 'o']\n",
            "Initialize PyTorch weight ['encoder', 'block_011', 'layer_000', 'SelfAttention', 'o']\n",
            "Skipping encoder/block_011/layer_000/SelfAttention/o_slot_vc\n",
            "Skipping encoder/block_011/layer_000/SelfAttention/o_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_011', 'layer_000', 'SelfAttention', 'q']\n",
            "Initialize PyTorch weight ['encoder', 'block_011', 'layer_000', 'SelfAttention', 'q']\n",
            "Skipping encoder/block_011/layer_000/SelfAttention/q_slot_vc\n",
            "Skipping encoder/block_011/layer_000/SelfAttention/q_slot_vr\n",
            "Transposing numpy weight of shape (1472, 384) for ['encoder', 'block_011', 'layer_000', 'SelfAttention', 'v']\n",
            "Initialize PyTorch weight ['encoder', 'block_011', 'layer_000', 'SelfAttention', 'v']\n",
            "Skipping encoder/block_011/layer_000/SelfAttention/v_slot_vc\n",
            "Skipping encoder/block_011/layer_000/SelfAttention/v_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_011', 'layer_000', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_011', 'layer_000', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_011/layer_000/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_011', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_011', 'layer_001', 'DenseReluDense', 'wi_0', 'kernel']\n",
            "Skipping encoder/block_011/layer_001/DenseReluDense/wi_0/kernel_slot_vc\n",
            "Skipping encoder/block_011/layer_001/DenseReluDense/wi_0/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472, 3584) for ['encoder', 'block_011', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_011', 'layer_001', 'DenseReluDense', 'wi_1', 'kernel']\n",
            "Skipping encoder/block_011/layer_001/DenseReluDense/wi_1/kernel_slot_vc\n",
            "Skipping encoder/block_011/layer_001/DenseReluDense/wi_1/kernel_slot_vr\n",
            "Transposing numpy weight of shape (3584, 1472) for ['encoder', 'block_011', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Initialize PyTorch weight ['encoder', 'block_011', 'layer_001', 'DenseReluDense', 'wo', 'kernel']\n",
            "Skipping encoder/block_011/layer_001/DenseReluDense/wo/kernel_slot_vc\n",
            "Skipping encoder/block_011/layer_001/DenseReluDense/wo/kernel_slot_vr\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'block_011', 'layer_001', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'block_011', 'layer_001', 'rms_norm', 'scale']\n",
            "Skipping encoder/block_011/layer_001/rms_norm/scale_slot_v\n",
            "Transposing numpy weight of shape (1472,) for ['encoder', 'rms_norm', 'scale']\n",
            "Initialize PyTorch weight ['encoder', 'rms_norm', 'scale']\n",
            "Skipping encoder/rms_norm/scale_slot_v\n",
            "Skipping global_step\n",
            "Initialize PyTorch weight ['shared', 'embedding']\n",
            "Skipping shared/embedding_slot_vc\n",
            "Skipping shared/embedding_slot_vr\n",
            "Weights not copied to PyTorch model: .\n",
            "Save PyTorch model to ./drive/MyDrive/mlin/dvcorpus/dv-t5/pytorch_model.bin\n",
            "Configuration saved in ./drive/MyDrive/mlin/dvcorpus/dv-t5/pytorch_model.bin/config.json\n",
            "Model weights saved in ./drive/MyDrive/mlin/dvcorpus/dv-t5/pytorch_model.bin/pytorch_model.bin\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zBCr-ZAblstS",
        "outputId": "b02b3198-dca7-4b39-cd89-3ba92b7da278"
      },
      "source": [
        "from transformers import TFT5ForConditionalGeneration\n",
        "t_model = TFT5ForConditionalGeneration.from_pretrained('./drive/MyDrive/mlin/dvcorpus/dv-t5', from_pt=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFT5ForConditionalGeneration: ['decoder.embed_tokens.weight', 'encoder.embed_tokens.weight']\n",
            "- This IS expected if you are initializing TFT5ForConditionalGeneration from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFT5ForConditionalGeneration from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the weights of TFT5ForConditionalGeneration were initialized from the PyTorch model.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFT5ForConditionalGeneration for predictions without further training.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0iF4G10ml-4Y"
      },
      "source": [
        "t_model.save_pretrained('./drive/MyDrive/mlin/dvcorpus/dv-t5/')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}