chapter11_part02_sequence-models.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0A49wbWHJvID"
      },
      "source": [
        "이 노트북은 [케라스 창시자에게 배우는 딥러닝 2판](https://tensorflow.blog/kerasdl2/)의 예제 코드를 담고 있습니다.\n",
        "\n",
        "<table align=\"left\">\n",
        "    <tr>\n",
        "        <td>\n",
        "            <a href=\"https://colab.research.google.com/github/rickiepark/deep-learning-with-python-2nd/blob/main/chapter11_part02_sequence-models.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
        "        </td>\n",
        "    </tr>\n",
        "</table>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5Xd_jva3JvIF"
      },
      "source": [
        "### 단어를 시퀀스로 처리하기: 시퀀스 모델 방식"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gJDW7N7JJvIG"
      },
      "source": [
        "#### 첫 번째 예제"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oBcqxVV3JvIG"
      },
      "source": [
        "**데이터 다운로드**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vM4VpJ7KJvIH",
        "outputId": "93a05378-32d0-4a31-e835-f1d83b4d6bee"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "rm: cannot remove 'aclImdb': No such file or directory\n",
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100 80.2M  100 80.2M    0     0  4916k      0  0:00:16  0:00:16 --:--:-- 8263k\n"
          ]
        }
      ],
      "source": [
        "!rm -r aclImdb\n",
        "!curl -O https://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\n",
        "!tar -xf aclImdb_v1.tar.gz\n",
        "!rm -r aclImdb/train/unsup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ztDHYG4gJvII"
      },
      "source": [
        "**데이터 준비**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "azorKPW0JvII",
        "outputId": "7aea2e81-10ce-4095-edec-56446f866c14"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found 20000 files belonging to 2 classes.\n",
            "Found 5000 files belonging to 2 classes.\n",
            "Found 25000 files belonging to 2 classes.\n"
          ]
        }
      ],
      "source": [
        "import os, pathlib, shutil, random\n",
        "from tensorflow import keras\n",
        "batch_size = 32\n",
        "base_dir = pathlib.Path(\"aclImdb\")\n",
        "val_dir = base_dir / \"val\"\n",
        "train_dir = base_dir / \"train\"\n",
        "\n",
        "for category in (\"neg\", \"pos\"):\n",
        "    os.makedirs(val_dir / category)\n",
        "    files = os.listdir(train_dir / category)\n",
        "    random.Random(1337).shuffle(files)\n",
        "    num_val_samples = int(0.2 * len(files))\n",
        "    val_files = files[-num_val_samples:]\n",
        "    for fname in val_files:\n",
        "        shutil.move(train_dir / category / fname,\n",
        "                    val_dir / category / fname)\n",
        "\n",
        "train_ds = keras.utils.text_dataset_from_directory(\n",
        "    \"aclImdb/train\", batch_size=batch_size\n",
        ")\n",
        "val_ds = keras.utils.text_dataset_from_directory(\n",
        "    \"aclImdb/val\", batch_size=batch_size\n",
        ")\n",
        "test_ds = keras.utils.text_dataset_from_directory(\n",
        "    \"aclImdb/test\", batch_size=batch_size\n",
        ")\n",
        "text_only_train_ds = train_ds.map(lambda x, y: x)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zz4tz-n4JvIJ"
      },
      "source": [
        "**정수 시퀀스 데이터셋 준비하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "iuPqw9cqJvIJ"
      },
      "outputs": [],
      "source": [
        "from tensorflow.keras import layers\n",
        "\n",
        "max_length = 600\n",
        "max_tokens = 20000\n",
        "text_vectorization = layers.TextVectorization(\n",
        "    max_tokens=max_tokens,\n",
        "    output_mode=\"int\",\n",
        "    output_sequence_length=max_length,\n",
        ")\n",
        "text_vectorization.adapt(text_only_train_ds)\n",
        "\n",
        "int_train_ds = train_ds.map(\n",
        "    lambda x, y: (text_vectorization(x), y),\n",
        "    num_parallel_calls=4)\n",
        "int_val_ds = val_ds.map(\n",
        "    lambda x, y: (text_vectorization(x), y),\n",
        "    num_parallel_calls=4)\n",
        "int_test_ds = test_ds.map(\n",
        "    lambda x, y: (text_vectorization(x), y),\n",
        "    num_parallel_calls=4)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7rCBxfAFJvIK"
      },
      "source": [
        "**원-핫 인코딩된 벡터 시퀀스로 시퀀스 모델 만들기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3rai-1TDJvIK",
        "outputId": "4e8f7943-564d-4629-d2fb-54c4d556a5b7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_1 (InputLayer)        [(None, None)]            0         \n",
            "                                                                 \n",
            " tf.one_hot (TFOpLambda)     (None, None, 20000)       0         \n",
            "                                                                 \n",
            " bidirectional (Bidirection  (None, 64)                5128448   \n",
            " al)                                                             \n",
            "                                                                 \n",
            " dropout (Dropout)           (None, 64)                0         \n",
            "                                                                 \n",
            " dense (Dense)               (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 5128513 (19.56 MB)\n",
            "Trainable params: 5128513 (19.56 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "import tensorflow as tf\n",
        "inputs = keras.Input(shape=(None,), dtype=\"int64\")\n",
        "embedded = tf.one_hot(inputs, depth=max_tokens)\n",
        "x = layers.Bidirectional(layers.LSTM(32))(embedded)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model = keras.Model(inputs, outputs)\n",
        "model.compile(optimizer=\"rmsprop\",\n",
        "              loss=\"binary_crossentropy\",\n",
        "              metrics=[\"accuracy\"])\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TeqpjAQoJvIL"
      },
      "source": [
        "**첫 번째 시퀀스 모델 훈련하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bl6MtP3NJvIL",
        "outputId": "05477967-9efd-4d38-d5b5-2ea2436d1b69"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "625/625 [==============================] - 60s 96ms/step - loss: 0.4053 - accuracy: 0.8417 - val_loss: 0.3337 - val_accuracy: 0.8776\n",
            "Epoch 2/10\n",
            "  1/625 [..............................] - ETA: 58s - loss: 0.3699 - accuracy: 0.8750"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py:3079: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
            "  saving_api.save_model(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "625/625 [==============================] - 60s 95ms/step - loss: 0.3114 - accuracy: 0.8871 - val_loss: 0.3029 - val_accuracy: 0.8908\n",
            "Epoch 3/10\n",
            "625/625 [==============================] - 60s 95ms/step - loss: 0.2707 - accuracy: 0.9071 - val_loss: 0.3504 - val_accuracy: 0.8510\n",
            "Epoch 4/10\n",
            "625/625 [==============================] - 60s 95ms/step - loss: 0.2336 - accuracy: 0.9218 - val_loss: 0.3676 - val_accuracy: 0.8836\n",
            "Epoch 5/10\n",
            "625/625 [==============================] - 60s 95ms/step - loss: 0.2042 - accuracy: 0.9313 - val_loss: 0.2786 - val_accuracy: 0.8920\n",
            "Epoch 6/10\n",
            "625/625 [==============================] - 60s 96ms/step - loss: 0.1811 - accuracy: 0.9420 - val_loss: 0.3031 - val_accuracy: 0.8924\n",
            "Epoch 7/10\n",
            "625/625 [==============================] - 60s 95ms/step - loss: 0.1615 - accuracy: 0.9481 - val_loss: 0.3695 - val_accuracy: 0.8890\n",
            "Epoch 8/10\n",
            "625/625 [==============================] - 60s 96ms/step - loss: 0.1442 - accuracy: 0.9547 - val_loss: 0.3390 - val_accuracy: 0.8924\n",
            "Epoch 9/10\n",
            "625/625 [==============================] - 60s 95ms/step - loss: 0.1261 - accuracy: 0.9610 - val_loss: 0.3416 - val_accuracy: 0.8808\n",
            "Epoch 10/10\n",
            "625/625 [==============================] - 60s 95ms/step - loss: 0.1054 - accuracy: 0.9676 - val_loss: 0.3487 - val_accuracy: 0.8842\n",
            "782/782 [==============================] - 33s 42ms/step - loss: 0.3112 - accuracy: 0.8799\n",
            "테스트 정확도: 0.880\n"
          ]
        }
      ],
      "source": [
        "callbacks = [\n",
        "    keras.callbacks.ModelCheckpoint(\"one_hot_bidir_lstm.h5\",\n",
        "                                    save_best_only=True)\n",
        "]\n",
        "model.fit(int_train_ds, validation_data=int_val_ds, epochs=10, callbacks=callbacks)\n",
        "model = keras.models.load_model(\"one_hot_bidir_lstm.h5\")\n",
        "print(f\"테스트 정확도: {model.evaluate(int_test_ds)[1]:.3f}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cVY_SwScJvIL"
      },
      "source": [
        "#### 단어 임베딩 이해하기"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "visueffQJvIL"
      },
      "source": [
        "#### 임베딩 층으로 단어 임베딩 학습하기"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rH2EhPHUJvIM"
      },
      "source": [
        "**`Embedding` 층 만들기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "LNACHsDAJvIM"
      },
      "outputs": [],
      "source": [
        "embedding_layer = layers.Embedding(input_dim=max_tokens, output_dim=256)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MfBr2Ld8JvIM"
      },
      "source": [
        "**밑바닥부터 훈련하는 `Embedding` 층을 사용한 모델**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NHbW8s3QJvIM",
        "outputId": "517b88b3-6edd-4176-957b-364ad2537217"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_1\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, None)]            0         \n",
            "                                                                 \n",
            " embedding_1 (Embedding)     (None, None, 256)         5120000   \n",
            "                                                                 \n",
            " bidirectional_1 (Bidirecti  (None, 64)                73984     \n",
            " onal)                                                           \n",
            "                                                                 \n",
            " dropout_1 (Dropout)         (None, 64)                0         \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 5194049 (19.81 MB)\n",
            "Trainable params: 5194049 (19.81 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n",
            "Epoch 1/10\n",
            "625/625 [==============================] - 49s 73ms/step - loss: 0.5405 - accuracy: 0.7225 - val_loss: 0.3821 - val_accuracy: 0.8494\n",
            "Epoch 2/10\n",
            "625/625 [==============================] - 31s 49ms/step - loss: 0.3661 - accuracy: 0.8599 - val_loss: 0.6173 - val_accuracy: 0.7780\n",
            "Epoch 3/10\n",
            "625/625 [==============================] - 27s 44ms/step - loss: 0.2890 - accuracy: 0.8920 - val_loss: 0.3103 - val_accuracy: 0.8826\n",
            "Epoch 4/10\n",
            "625/625 [==============================] - 24s 39ms/step - loss: 0.2434 - accuracy: 0.9155 - val_loss: 0.4969 - val_accuracy: 0.8402\n",
            "Epoch 5/10\n",
            "625/625 [==============================] - 24s 38ms/step - loss: 0.2125 - accuracy: 0.9270 - val_loss: 0.3392 - val_accuracy: 0.8866\n",
            "Epoch 6/10\n",
            "625/625 [==============================] - 24s 38ms/step - loss: 0.1832 - accuracy: 0.9378 - val_loss: 0.3485 - val_accuracy: 0.8772\n",
            "Epoch 7/10\n",
            "625/625 [==============================] - 23s 37ms/step - loss: 0.1556 - accuracy: 0.9488 - val_loss: 0.3623 - val_accuracy: 0.8604\n",
            "Epoch 8/10\n",
            "625/625 [==============================] - 23s 36ms/step - loss: 0.1321 - accuracy: 0.9569 - val_loss: 0.4931 - val_accuracy: 0.8418\n",
            "Epoch 9/10\n",
            "625/625 [==============================] - 22s 35ms/step - loss: 0.1182 - accuracy: 0.9627 - val_loss: 0.4107 - val_accuracy: 0.8720\n",
            "Epoch 10/10\n",
            "625/625 [==============================] - 22s 36ms/step - loss: 0.1003 - accuracy: 0.9688 - val_loss: 0.4704 - val_accuracy: 0.8694\n",
            "782/782 [==============================] - 12s 14ms/step - loss: 0.3433 - accuracy: 0.8672\n",
            "테스트 정확도: 0.867\n"
          ]
        }
      ],
      "source": [
        "inputs = keras.Input(shape=(None,), dtype=\"int64\")\n",
        "embedded = layers.Embedding(input_dim=max_tokens, output_dim=256)(inputs)\n",
        "x = layers.Bidirectional(layers.LSTM(32))(embedded)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model = keras.Model(inputs, outputs)\n",
        "model.compile(optimizer=\"rmsprop\",\n",
        "              loss=\"binary_crossentropy\",\n",
        "              metrics=[\"accuracy\"])\n",
        "model.summary()\n",
        "\n",
        "callbacks = [\n",
        "    keras.callbacks.ModelCheckpoint(\"embeddings_bidir_lstm.h5\",\n",
        "                                    save_best_only=True)\n",
        "]\n",
        "model.fit(int_train_ds, validation_data=int_val_ds, epochs=10, callbacks=callbacks)\n",
        "model = keras.models.load_model(\"embeddings_bidir_lstm.h5\")\n",
        "print(f\"테스트 정확도: {model.evaluate(int_test_ds)[1]:.3f}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W32FVDl_JvIM"
      },
      "source": [
        "#### 패딩과 마스킹 이해하기"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M37QJPqIJvIN"
      },
      "source": [
        "**마스킹을 활성화한 `Embedding` 층 사용하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VS7crehBJvIN",
        "outputId": "7a830178-df96-4c15-90e9-4c13bfdac9a7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_3 (InputLayer)        [(None, None)]            0         \n",
            "                                                                 \n",
            " embedding_2 (Embedding)     (None, None, 256)         5120000   \n",
            "                                                                 \n",
            " bidirectional_2 (Bidirecti  (None, 64)                73984     \n",
            " onal)                                                           \n",
            "                                                                 \n",
            " dropout_2 (Dropout)         (None, 64)                0         \n",
            "                                                                 \n",
            " dense_2 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 5194049 (19.81 MB)\n",
            "Trainable params: 5194049 (19.81 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n",
            "Epoch 1/10\n",
            "625/625 [==============================] - 61s 85ms/step - loss: 0.4478 - accuracy: 0.7812 - val_loss: 0.3441 - val_accuracy: 0.8432\n",
            "Epoch 2/10\n",
            "625/625 [==============================] - 35s 57ms/step - loss: 0.2830 - accuracy: 0.8892 - val_loss: 0.2683 - val_accuracy: 0.8924\n",
            "Epoch 3/10\n",
            "625/625 [==============================] - 31s 50ms/step - loss: 0.2140 - accuracy: 0.9180 - val_loss: 0.2712 - val_accuracy: 0.8884\n",
            "Epoch 4/10\n",
            "625/625 [==============================] - 28s 45ms/step - loss: 0.1672 - accuracy: 0.9380 - val_loss: 0.3031 - val_accuracy: 0.8934\n",
            "Epoch 5/10\n",
            "625/625 [==============================] - 27s 43ms/step - loss: 0.1264 - accuracy: 0.9549 - val_loss: 0.3047 - val_accuracy: 0.8896\n",
            "Epoch 6/10\n",
            "625/625 [==============================] - 27s 43ms/step - loss: 0.0979 - accuracy: 0.9665 - val_loss: 0.4195 - val_accuracy: 0.8744\n",
            "Epoch 7/10\n",
            "625/625 [==============================] - 27s 43ms/step - loss: 0.0779 - accuracy: 0.9735 - val_loss: 0.3566 - val_accuracy: 0.8794\n",
            "Epoch 8/10\n",
            "625/625 [==============================] - 27s 43ms/step - loss: 0.0561 - accuracy: 0.9819 - val_loss: 0.4386 - val_accuracy: 0.8832\n",
            "Epoch 9/10\n",
            "625/625 [==============================] - 27s 43ms/step - loss: 0.0478 - accuracy: 0.9847 - val_loss: 0.5236 - val_accuracy: 0.8796\n",
            "Epoch 10/10\n",
            "625/625 [==============================] - 26s 42ms/step - loss: 0.0367 - accuracy: 0.9883 - val_loss: 0.4945 - val_accuracy: 0.8808\n",
            "782/782 [==============================] - 15s 16ms/step - loss: 0.3016 - accuracy: 0.8789\n",
            "테스트 정확도: 0.879\n"
          ]
        }
      ],
      "source": [
        "inputs = keras.Input(shape=(None,), dtype=\"int64\")\n",
        "embedded = layers.Embedding(\n",
        "    input_dim=max_tokens, output_dim=256, mask_zero=True)(inputs)\n",
        "x = layers.Bidirectional(layers.LSTM(32))(embedded)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model = keras.Model(inputs, outputs)\n",
        "model.compile(optimizer=\"rmsprop\",\n",
        "              loss=\"binary_crossentropy\",\n",
        "              metrics=[\"accuracy\"])\n",
        "model.summary()\n",
        "\n",
        "callbacks = [\n",
        "    keras.callbacks.ModelCheckpoint(\"embeddings_bidir_lstm_with_masking.h5\",\n",
        "                                    save_best_only=True)\n",
        "]\n",
        "model.fit(int_train_ds, validation_data=int_val_ds, epochs=10, callbacks=callbacks)\n",
        "model = keras.models.load_model(\"embeddings_bidir_lstm_with_masking.h5\")\n",
        "print(f\"테스트 정확도: {model.evaluate(int_test_ds)[1]:.3f}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WT_UyLakJvIN"
      },
      "source": [
        "#### 사전 훈련된 단어 임베딩 사용하기"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QST9q6Q1JvIN",
        "outputId": "d484f896-16a8-4920-d47f-4b380016b9e8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-11-04 03:32:09--  http://nlp.stanford.edu/data/glove.6B.zip\n",
            "Resolving nlp.stanford.edu (nlp.stanford.edu)... 171.64.67.140\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:80... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://nlp.stanford.edu/data/glove.6B.zip [following]\n",
            "--2023-11-04 03:32:09--  https://nlp.stanford.edu/data/glove.6B.zip\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:443... connected.\n",
            "HTTP request sent, awaiting response... 301 Moved Permanently\n",
            "Location: https://downloads.cs.stanford.edu/nlp/data/glove.6B.zip [following]\n",
            "--2023-11-04 03:32:09--  https://downloads.cs.stanford.edu/nlp/data/glove.6B.zip\n",
            "Resolving downloads.cs.stanford.edu (downloads.cs.stanford.edu)... 171.64.64.22\n",
            "Connecting to downloads.cs.stanford.edu (downloads.cs.stanford.edu)|171.64.64.22|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 862182613 (822M) [application/zip]\n",
            "Saving to: ‘glove.6B.zip’\n",
            "\n",
            "glove.6B.zip        100%[===================>] 822.24M  5.11MB/s    in 2m 40s  \n",
            "\n",
            "2023-11-04 03:34:50 (5.15 MB/s) - ‘glove.6B.zip’ saved [862182613/862182613]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget http://nlp.stanford.edu/data/glove.6B.zip\n",
        "!unzip -q glove.6B.zip"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t0aUgmK2JvIN"
      },
      "source": [
        "**GloVe 단어 임베딩 파일 파싱하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xZwkvoGYJvIN",
        "outputId": "3c392e8d-27ae-4ff2-bb5b-317d31a5802f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "단어 벡터 개수: 400000\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "path_to_glove_file = \"glove.6B.100d.txt\"\n",
        "\n",
        "embeddings_index = {}\n",
        "with open(path_to_glove_file) as f:\n",
        "    for line in f:\n",
        "        word, coefs = line.split(maxsplit=1)\n",
        "        coefs = np.fromstring(coefs, \"f\", sep=\" \")\n",
        "        embeddings_index[word] = coefs\n",
        "\n",
        "print(f\"단어 벡터 개수: {len(embeddings_index)}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hCdliBn7JvIO"
      },
      "source": [
        "**GloVe 단어 임베딩 행렬 준비하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "HTEMsvf2JvIO"
      },
      "outputs": [],
      "source": [
        "embedding_dim = 100\n",
        "\n",
        "vocabulary = text_vectorization.get_vocabulary()\n",
        "word_index = dict(zip(vocabulary, range(len(vocabulary))))\n",
        "\n",
        "embedding_matrix = np.zeros((max_tokens, embedding_dim))\n",
        "for word, i in word_index.items():\n",
        "    if i < max_tokens:\n",
        "        embedding_vector = embeddings_index.get(word)\n",
        "    if embedding_vector is not None:\n",
        "        embedding_matrix[i] = embedding_vector"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "PmEvrFkYJvIO"
      },
      "outputs": [],
      "source": [
        "embedding_layer = layers.Embedding(\n",
        "    max_tokens,\n",
        "    embedding_dim,\n",
        "    embeddings_initializer=keras.initializers.Constant(embedding_matrix),\n",
        "    trainable=False,\n",
        "    mask_zero=True,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m24hC78pJvIO"
      },
      "source": [
        "**사전 훈련된 임베딩을 사용하는 모델**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CAdQbaClJvIO",
        "outputId": "0b79788f-ffa6-4f95-e887-491929c58f46"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_3\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_4 (InputLayer)        [(None, None)]            0         \n",
            "                                                                 \n",
            " embedding_3 (Embedding)     (None, None, 100)         2000000   \n",
            "                                                                 \n",
            " bidirectional_3 (Bidirecti  (None, 64)                34048     \n",
            " onal)                                                           \n",
            "                                                                 \n",
            " dropout_3 (Dropout)         (None, 64)                0         \n",
            "                                                                 \n",
            " dense_3 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 2034113 (7.76 MB)\n",
            "Trainable params: 34113 (133.25 KB)\n",
            "Non-trainable params: 2000000 (7.63 MB)\n",
            "_________________________________________________________________\n",
            "Epoch 1/10\n",
            "625/625 [==============================] - 43s 56ms/step - loss: 0.5839 - accuracy: 0.6873 - val_loss: 0.4812 - val_accuracy: 0.7706\n",
            "Epoch 2/10\n",
            "625/625 [==============================] - 30s 49ms/step - loss: 0.4530 - accuracy: 0.7933 - val_loss: 0.4199 - val_accuracy: 0.8090\n",
            "Epoch 3/10\n",
            "625/625 [==============================] - 30s 47ms/step - loss: 0.4024 - accuracy: 0.8238 - val_loss: 0.3591 - val_accuracy: 0.8446\n",
            "Epoch 4/10\n",
            "625/625 [==============================] - 30s 48ms/step - loss: 0.3596 - accuracy: 0.8460 - val_loss: 0.3399 - val_accuracy: 0.8570\n",
            "Epoch 5/10\n",
            "625/625 [==============================] - 26s 41ms/step - loss: 0.3373 - accuracy: 0.8587 - val_loss: 0.3678 - val_accuracy: 0.8436\n",
            "Epoch 6/10\n",
            "625/625 [==============================] - 30s 47ms/step - loss: 0.3175 - accuracy: 0.8669 - val_loss: 0.3374 - val_accuracy: 0.8582\n",
            "Epoch 7/10\n",
            "625/625 [==============================] - 29s 46ms/step - loss: 0.2990 - accuracy: 0.8762 - val_loss: 0.3182 - val_accuracy: 0.8646\n",
            "Epoch 8/10\n",
            "625/625 [==============================] - 29s 46ms/step - loss: 0.2819 - accuracy: 0.8845 - val_loss: 0.3024 - val_accuracy: 0.8740\n",
            "Epoch 9/10\n",
            "625/625 [==============================] - 29s 46ms/step - loss: 0.2658 - accuracy: 0.8912 - val_loss: 0.2979 - val_accuracy: 0.8788\n",
            "Epoch 10/10\n",
            "625/625 [==============================] - 24s 38ms/step - loss: 0.2567 - accuracy: 0.8984 - val_loss: 0.3026 - val_accuracy: 0.8826\n"
          ]
        }
      ],
      "source": [
        "inputs = keras.Input(shape=(None,), dtype=\"int64\")\n",
        "embedded = embedding_layer(inputs)\n",
        "x = layers.Bidirectional(layers.LSTM(32))(embedded)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model = keras.Model(inputs, outputs)\n",
        "model.compile(optimizer=\"rmsprop\",\n",
        "              loss=\"binary_crossentropy\",\n",
        "              metrics=[\"accuracy\"])\n",
        "model.summary()\n",
        "\n",
        "callbacks = [\n",
        "    keras.callbacks.ModelCheckpoint(\"glove_embeddings_sequence_model.h5\",\n",
        "                                    save_best_only=True)\n",
        "]\n",
        "model.fit(int_train_ds, validation_data=int_val_ds, epochs=10, callbacks=callbacks)\n",
        "model = keras.models.load_model(\"glove_embeddings_sequence_model.h5\")\n",
        "print(f\"테스트 정확도: {model.evaluate(int_test_ds)[1]:.3f}\")"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "chapter11_part02_sequence-models.i",
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "V100",
      "gpuClass": "premium"
    },
    "kernelspec": {
      "display_name": "default:Python",
      "language": "python",
      "name": "conda-env-default-py"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.10"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}