chapter11_part01_introduction.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "khWcofDMPHV9"
      },
      "source": [
        "이 노트북은 [케라스 창시자에게 배우는 딥러닝 2판](https://tensorflow.blog/kerasdl2/)의 예제 코드를 담고 있습니다.\n",
        "\n",
        "<table align=\"left\">\n",
        "    <tr>\n",
        "        <td>\n",
        "            <a href=\"https://colab.research.google.com/github/rickiepark/deep-learning-with-python-2nd/blob/main/chapter11_part01_introduction.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
        "        </td>\n",
        "    </tr>\n",
        "</table>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8Ce2SFsxPHWA"
      },
      "source": [
        "# 텍스트를 위한 딥러닝"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fN2ZLGlaPHWA"
      },
      "source": [
        "## 자연어 처리 소개"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6b--VkdzPHWB"
      },
      "source": [
        "## 텍스트 데이터 준비"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VEToG7w2PHWB"
      },
      "source": [
        "### 텍스트 표준화"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oWyNOPSKPHWB"
      },
      "source": [
        "### 텍스트 분할 (토큰화)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5h-ecuq5PHWB"
      },
      "source": [
        "### 어휘 인덱싱"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vThLgagxPHWC"
      },
      "source": [
        "### `TextVectorization` 층 사용하기"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "e6AsY6CHPHWC"
      },
      "outputs": [],
      "source": [
        "import string\n",
        "\n",
        "class Vectorizer:\n",
        "    def standardize(self, text):\n",
        "        text = text.lower()\n",
        "        return \"\".join(char for char in text if char not in string.punctuation)\n",
        "\n",
        "    def tokenize(self, text):\n",
        "        return text.split()\n",
        "\n",
        "    def make_vocabulary(self, dataset):\n",
        "        self.vocabulary = {\"\": 0, \"[UNK]\": 1}\n",
        "        for text in dataset:\n",
        "            text = self.standardize(text)\n",
        "            tokens = self.tokenize(text)\n",
        "            for token in tokens:\n",
        "                if token not in self.vocabulary:\n",
        "                    self.vocabulary[token] = len(self.vocabulary)\n",
        "        self.inverse_vocabulary = dict(\n",
        "            (v, k) for k, v in self.vocabulary.items())\n",
        "\n",
        "    def encode(self, text):\n",
        "        text = self.standardize(text)\n",
        "        tokens = self.tokenize(text)\n",
        "        return [self.vocabulary.get(token, 1) for token in tokens]\n",
        "\n",
        "    def decode(self, int_sequence):\n",
        "        return \" \".join(\n",
        "            self.inverse_vocabulary.get(i, \"[UNK]\") for i in int_sequence)\n",
        "\n",
        "vectorizer = Vectorizer()\n",
        "dataset = [\n",
        "    \"I write, erase, rewrite\",\n",
        "    \"Erase again, and then\",\n",
        "    \"A poppy blooms.\",\n",
        "]\n",
        "vectorizer.make_vocabulary(dataset)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "qinD6gJ1PHWE",
        "outputId": "ba9326e2-7b33-4cdb-d0ed-e674a601d05f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[2, 3, 5, 7, 1, 5, 6]\n"
          ]
        }
      ],
      "source": [
        "test_sentence = \"I write, rewrite, and still rewrite again\"\n",
        "encoded_sentence = vectorizer.encode(test_sentence)\n",
        "print(encoded_sentence)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "lTIHl1YNPHWF",
        "outputId": "19933849-c4e9-4b38-d50e-ff95732e2a54",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "i write rewrite and [UNK] rewrite again\n"
          ]
        }
      ],
      "source": [
        "decoded_sentence = vectorizer.decode(encoded_sentence)\n",
        "print(decoded_sentence)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "COXYu3qdPHWF"
      },
      "outputs": [],
      "source": [
        "from tensorflow.keras.layers import TextVectorization\n",
        "text_vectorization = TextVectorization(\n",
        "    output_mode=\"int\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "5OWCEbFEPHWF"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "import string\n",
        "import tensorflow as tf\n",
        "\n",
        "def custom_standardization_fn(string_tensor):\n",
        "    lowercase_string = tf.strings.lower(string_tensor)\n",
        "    return tf.strings.regex_replace(\n",
        "        lowercase_string, f\"[{re.escape(string.punctuation)}]\", \"\")\n",
        "\n",
        "def custom_split_fn(string_tensor):\n",
        "    return tf.strings.split(string_tensor)\n",
        "\n",
        "text_vectorization = TextVectorization(\n",
        "    output_mode=\"int\",\n",
        "    standardize=custom_standardization_fn,\n",
        "    split=custom_split_fn,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "a8KKZ2aGPHWG"
      },
      "outputs": [],
      "source": [
        "dataset = [\n",
        "    \"I write, erase, rewrite\",\n",
        "    \"Erase again, and then\",\n",
        "    \"A poppy blooms.\",\n",
        "]\n",
        "text_vectorization.adapt(dataset)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ivDt8LVRPHWG"
      },
      "source": [
        "**어휘 사전 출력하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "6piMhRMEPHWG",
        "outputId": "0bd7e483-df54-47e6-870b-3c870b39164b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['',\n",
              " '[UNK]',\n",
              " 'erase',\n",
              " 'write',\n",
              " 'then',\n",
              " 'rewrite',\n",
              " 'poppy',\n",
              " 'i',\n",
              " 'blooms',\n",
              " 'and',\n",
              " 'again',\n",
              " 'a']"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ],
      "source": [
        "text_vectorization.get_vocabulary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "ZQj8w725PHWG",
        "outputId": "4c949fb5-22ee-42b6-bdd7-c4b5f14575b1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tf.Tensor([ 7  3  5  9  1  5 10], shape=(7,), dtype=int64)\n"
          ]
        }
      ],
      "source": [
        "vocabulary = text_vectorization.get_vocabulary()\n",
        "test_sentence = \"I write, rewrite, and still rewrite again\"\n",
        "encoded_sentence = text_vectorization(test_sentence)\n",
        "print(encoded_sentence)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "2ACo5XXcPHWG",
        "outputId": "1172623e-1b42-4db0-db65-0d751cfce9fd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "i write rewrite and [UNK] rewrite again\n"
          ]
        }
      ],
      "source": [
        "inverse_vocab = dict(enumerate(vocabulary))\n",
        "decoded_sentence = \" \".join(inverse_vocab[int(i)] for i in encoded_sentence)\n",
        "print(decoded_sentence)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VjnG_PtiPHWH"
      },
      "source": [
        "## 단어 그룹을 표현하는 두 가지 방법: 집합과 시퀀스"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KFQeM-FUPHWH"
      },
      "source": [
        "### IMDB 영화 리뷰 데이터 준비하기"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "JHC5wgjxPHWH",
        "outputId": "2f6a0e90-55c2-4e1b-be3f-277ef5646d99",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100 80.2M  100 80.2M    0     0  29.6M      0  0:00:02  0:00:02 --:--:-- 29.6M\n"
          ]
        }
      ],
      "source": [
        "!curl -O https://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\n",
        "!tar -xf aclImdb_v1.tar.gz"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "irDrd9GJPHWH"
      },
      "outputs": [],
      "source": [
        "!rm -r aclImdb/train/unsup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "pPt42d2WPHWH",
        "outputId": "11af3218-2571-407e-9765-96ba1b8b0d42",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "I first saw this back in the early 90s on UK TV, i did like it then but i missed the chance to tape it, many years passed but the film always stuck with me and i lost hope of seeing it TV again, the main thing that stuck with me was the end, the hole castle part really touched me, its easy to watch, has a great story, great music, the list goes on and on, its OK me saying how good it is but everyone will take there own best bits away with them once they have seen it, yes the animation is top notch and beautiful to watch, it does show its age in a very few parts but that has now become part of it beauty, i am so glad it has came out on DVD as it is one of my top 10 films of all time. Buy it or rent it just see it, best viewing is at night alone with drink and food in reach so you don't have to stop the film.<br /><br />Enjoy"
          ]
        }
      ],
      "source": [
        "!cat aclImdb/train/pos/4077_10.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "VqMxuzj8PHWI"
      },
      "outputs": [],
      "source": [
        "import os, pathlib, shutil, random\n",
        "\n",
        "base_dir = pathlib.Path(\"aclImdb\")\n",
        "val_dir = base_dir / \"val\"\n",
        "train_dir = base_dir / \"train\"\n",
        "for category in (\"neg\", \"pos\"):\n",
        "    os.makedirs(val_dir / category)\n",
        "    files = os.listdir(train_dir / category)\n",
        "    random.Random(1337).shuffle(files)\n",
        "    num_val_samples = int(0.2 * len(files))\n",
        "    val_files = files[-num_val_samples:]\n",
        "    for fname in val_files:\n",
        "        shutil.move(train_dir / category / fname,\n",
        "                    val_dir / category / fname)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "nCA9IoOKPHWI",
        "outputId": "8645a265-4443-4496-859b-d410107838fa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found 20000 files belonging to 2 classes.\n",
            "Found 5000 files belonging to 2 classes.\n",
            "Found 25000 files belonging to 2 classes.\n"
          ]
        }
      ],
      "source": [
        "from tensorflow import keras\n",
        "batch_size = 32\n",
        "\n",
        "train_ds = keras.utils.text_dataset_from_directory(\n",
        "    \"aclImdb/train\", batch_size=batch_size\n",
        ")\n",
        "val_ds = keras.utils.text_dataset_from_directory(\n",
        "    \"aclImdb/val\", batch_size=batch_size\n",
        ")\n",
        "test_ds = keras.utils.text_dataset_from_directory(\n",
        "    \"aclImdb/test\", batch_size=batch_size\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F6IhmydcPHWI"
      },
      "source": [
        "**첫 번째 배치의 크기와 dtype 출력하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "RO58kApSPHWI",
        "outputId": "62ec02b8-dee9-4c7d-b005-a93d840c8638",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "inputs.shape: (32,)\n",
            "inputs.dtype: <dtype: 'string'>\n",
            "targets.shape: (32,)\n",
            "targets.dtype: <dtype: 'int32'>\n",
            "inputs[0]: tf.Tensor(b\"After you see Vertigo, then watch Bell, Book and Candle, made within months of each other.<br /><br />My second favorite Kim Novak film, with Picnic, coming in as third.<br /><br />All three performances are great, Vertigo, being the best, of all.<br /><br />They came to my nowhere Kansas Prairie town, near by, at Salina, Kansas in the 50s, to film, Picnic. <br /><br />Bell, Book and Candle's musical score, I believe is one of Alex North's. Perfect for this bit of comedy.<br /><br />After Vertigo, Stewart and Novak, did this comedy, how amusing to note the dramatic contrast.<br /><br />Worth your time, if you like Kim Novak. The Greta Garbo of my youth.\", shape=(), dtype=string)\n",
            "targets[0]: tf.Tensor(1, shape=(), dtype=int32)\n"
          ]
        }
      ],
      "source": [
        "for inputs, targets in train_ds:\n",
        "    print(\"inputs.shape:\", inputs.shape)\n",
        "    print(\"inputs.dtype:\", inputs.dtype)\n",
        "    print(\"targets.shape:\", targets.shape)\n",
        "    print(\"targets.dtype:\", targets.dtype)\n",
        "    print(\"inputs[0]:\", inputs[0])\n",
        "    print(\"targets[0]:\", targets[0])\n",
        "    break"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hRRDlnqAPHWJ"
      },
      "source": [
        "### 단어를 집합으로 처리하기: BoW 방식"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uFc9dZA5PHWJ"
      },
      "source": [
        "#### Single words (unigrams) with binary encoding"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r4_cyPXEPHWJ"
      },
      "source": [
        "**`TextVectorization` 층으로 데이터 전처리하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "eOSiAiGPPHWJ"
      },
      "outputs": [],
      "source": [
        "text_vectorization = TextVectorization(\n",
        "    max_tokens=20000,\n",
        "    output_mode=\"multi_hot\",\n",
        ")\n",
        "text_only_train_ds = train_ds.map(lambda x, y: x)\n",
        "text_vectorization.adapt(text_only_train_ds)\n",
        "\n",
        "binary_1gram_train_ds = train_ds.map(\n",
        "    lambda x, y: (text_vectorization(x), y),\n",
        "    num_parallel_calls=4)\n",
        "binary_1gram_val_ds = val_ds.map(\n",
        "    lambda x, y: (text_vectorization(x), y),\n",
        "    num_parallel_calls=4)\n",
        "binary_1gram_test_ds = test_ds.map(\n",
        "    lambda x, y: (text_vectorization(x), y),\n",
        "    num_parallel_calls=4)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XZp__LNzPHWJ"
      },
      "source": [
        "**이진 유니그램 데이터셋의 출력 확인하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "aKTPekqmPHWJ",
        "outputId": "54e23da8-7298-4388-848f-3ecc59d4c07f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "inputs.shape: (32, 20000)\n",
            "inputs.dtype: <dtype: 'float32'>\n",
            "targets.shape: (32,)\n",
            "targets.dtype: <dtype: 'int32'>\n",
            "inputs[0]: tf.Tensor([1. 1. 1. ... 0. 0. 0.], shape=(20000,), dtype=float32)\n",
            "targets[0]: tf.Tensor(1, shape=(), dtype=int32)\n"
          ]
        }
      ],
      "source": [
        "for inputs, targets in binary_1gram_train_ds:\n",
        "    print(\"inputs.shape:\", inputs.shape)\n",
        "    print(\"inputs.dtype:\", inputs.dtype)\n",
        "    print(\"targets.shape:\", targets.shape)\n",
        "    print(\"targets.dtype:\", targets.dtype)\n",
        "    print(\"inputs[0]:\", inputs[0])\n",
        "    print(\"targets[0]:\", targets[0])\n",
        "    break"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-zCVeBuAPHWJ"
      },
      "source": [
        "**모델 생성 유틸리티**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "ODGXYjUcPHWK"
      },
      "outputs": [],
      "source": [
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "\n",
        "def get_model(max_tokens=20000, hidden_dim=16):\n",
        "    inputs = keras.Input(shape=(max_tokens,))\n",
        "    x = layers.Dense(hidden_dim, activation=\"relu\")(inputs)\n",
        "    x = layers.Dropout(0.5)(x)\n",
        "    outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "    model = keras.Model(inputs, outputs)\n",
        "    model.compile(optimizer=\"rmsprop\",\n",
        "                  loss=\"binary_crossentropy\",\n",
        "                  metrics=[\"accuracy\"])\n",
        "    return model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R2vbysS9PHWK"
      },
      "source": [
        "**이진 유니그램 모델 훈련하고 테스트하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "Fmb0uHkyPHWK",
        "outputId": "73474e3e-8aaa-4c2e-e698-e0d570633d32",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_3 (InputLayer)        [(None, 20000)]           0         \n",
            "                                                                 \n",
            " dense_4 (Dense)             (None, 16)                320016    \n",
            "                                                                 \n",
            " dropout_2 (Dropout)         (None, 16)                0         \n",
            "                                                                 \n",
            " dense_5 (Dense)             (None, 1)                 17        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 320033 (1.22 MB)\n",
            "Trainable params: 320033 (1.22 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n",
            "Epoch 1/10\n",
            "625/625 [==============================] - 11s 14ms/step - loss: 0.4008 - accuracy: 0.8306 - val_loss: 0.2689 - val_accuracy: 0.8946\n",
            "Epoch 2/10\n",
            "625/625 [==============================] - 4s 7ms/step - loss: 0.2684 - accuracy: 0.9008 - val_loss: 0.2658 - val_accuracy: 0.8966\n",
            "Epoch 3/10\n",
            "625/625 [==============================] - 4s 6ms/step - loss: 0.2433 - accuracy: 0.9173 - val_loss: 0.2769 - val_accuracy: 0.8952\n",
            "Epoch 4/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.2227 - accuracy: 0.9260 - val_loss: 0.2940 - val_accuracy: 0.8970\n",
            "Epoch 5/10\n",
            "625/625 [==============================] - 4s 6ms/step - loss: 0.2148 - accuracy: 0.9292 - val_loss: 0.3039 - val_accuracy: 0.8980\n",
            "Epoch 6/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.1994 - accuracy: 0.9357 - val_loss: 0.3194 - val_accuracy: 0.8946\n",
            "Epoch 7/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.1983 - accuracy: 0.9366 - val_loss: 0.3341 - val_accuracy: 0.8938\n",
            "Epoch 8/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.2044 - accuracy: 0.9363 - val_loss: 0.3369 - val_accuracy: 0.8920\n",
            "Epoch 9/10\n",
            "625/625 [==============================] - 3s 5ms/step - loss: 0.1982 - accuracy: 0.9366 - val_loss: 0.3451 - val_accuracy: 0.8946\n",
            "Epoch 10/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.1967 - accuracy: 0.9387 - val_loss: 0.3543 - val_accuracy: 0.8900\n",
            "782/782 [==============================] - 5s 6ms/step - loss: 0.2879 - accuracy: 0.8870\n",
            "테스트 정확도: 0.887\n"
          ]
        }
      ],
      "source": [
        "model = get_model()\n",
        "model.summary()\n",
        "callbacks = [\n",
        "    keras.callbacks.ModelCheckpoint(\"binary_1gram.h5\",\n",
        "                                    save_best_only=True)\n",
        "]\n",
        "model.fit(binary_1gram_train_ds.cache(),\n",
        "          validation_data=binary_1gram_val_ds.cache(),\n",
        "          epochs=10,\n",
        "          callbacks=callbacks)\n",
        "model = keras.models.load_model(\"binary_1gram.h5\")\n",
        "print(f\"테스트 정확도: {model.evaluate(binary_1gram_test_ds)[1]:.3f}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tbLjMnm6PHWK"
      },
      "source": [
        "#### 이진 인코딩을 사용한 바이그램"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HqKkpvj3PHWK"
      },
      "source": [
        "**바이그램을 반환하는 `TextVectorization` 층 만들기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "O_xsjJvdPHWK"
      },
      "outputs": [],
      "source": [
        "text_vectorization = TextVectorization(\n",
        "    ngrams=2,\n",
        "    max_tokens=20000,\n",
        "    output_mode=\"multi_hot\",\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mdWi9dhXPHWL"
      },
      "source": [
        "**이진 바이그램 모델 훈련하고 테스트하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "7hJTKCooPHWL",
        "outputId": "64cdeea2-b898-4521-fcd7-189856febcba",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_3\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_4 (InputLayer)        [(None, 20000)]           0         \n",
            "                                                                 \n",
            " dense_6 (Dense)             (None, 16)                320016    \n",
            "                                                                 \n",
            " dropout_3 (Dropout)         (None, 16)                0         \n",
            "                                                                 \n",
            " dense_7 (Dense)             (None, 1)                 17        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 320033 (1.22 MB)\n",
            "Trainable params: 320033 (1.22 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n",
            "Epoch 1/10\n",
            "625/625 [==============================] - 8s 12ms/step - loss: 0.3901 - accuracy: 0.8371 - val_loss: 0.2620 - val_accuracy: 0.8972\n",
            "Epoch 2/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.2427 - accuracy: 0.9156 - val_loss: 0.2731 - val_accuracy: 0.8982\n",
            "Epoch 3/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.2048 - accuracy: 0.9324 - val_loss: 0.2816 - val_accuracy: 0.8994\n",
            "Epoch 4/10\n",
            "625/625 [==============================] - 4s 6ms/step - loss: 0.1827 - accuracy: 0.9431 - val_loss: 0.3161 - val_accuracy: 0.8928\n",
            "Epoch 5/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.1731 - accuracy: 0.9486 - val_loss: 0.3208 - val_accuracy: 0.8972\n",
            "Epoch 6/10\n",
            "625/625 [==============================] - 3s 5ms/step - loss: 0.1707 - accuracy: 0.9522 - val_loss: 0.3417 - val_accuracy: 0.8950\n",
            "Epoch 7/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.1653 - accuracy: 0.9538 - val_loss: 0.3647 - val_accuracy: 0.8944\n",
            "Epoch 8/10\n",
            "625/625 [==============================] - 3s 5ms/step - loss: 0.1684 - accuracy: 0.9539 - val_loss: 0.3673 - val_accuracy: 0.8924\n",
            "Epoch 9/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.1520 - accuracy: 0.9564 - val_loss: 0.3775 - val_accuracy: 0.8892\n",
            "Epoch 10/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.1563 - accuracy: 0.9570 - val_loss: 0.3972 - val_accuracy: 0.8890\n",
            "782/782 [==============================] - 7s 8ms/step - loss: 0.2666 - accuracy: 0.8964\n",
            "테스트 정확도: 0.896\n"
          ]
        }
      ],
      "source": [
        "text_vectorization.adapt(text_only_train_ds)\n",
        "binary_2gram_train_ds = train_ds.map(\n",
        "    lambda x, y: (text_vectorization(x), y),\n",
        "    num_parallel_calls=4)\n",
        "binary_2gram_val_ds = val_ds.map(\n",
        "    lambda x, y: (text_vectorization(x), y),\n",
        "    num_parallel_calls=4)\n",
        "binary_2gram_test_ds = test_ds.map(\n",
        "    lambda x, y: (text_vectorization(x), y),\n",
        "    num_parallel_calls=4)\n",
        "\n",
        "model = get_model()\n",
        "model.summary()\n",
        "callbacks = [\n",
        "    keras.callbacks.ModelCheckpoint(\"binary_2gram.h5\",\n",
        "                                    save_best_only=True)\n",
        "]\n",
        "model.fit(binary_2gram_train_ds.cache(),\n",
        "          validation_data=binary_2gram_val_ds.cache(),\n",
        "          epochs=10,\n",
        "          callbacks=callbacks)\n",
        "model = keras.models.load_model(\"binary_2gram.h5\")\n",
        "print(f\"테스트 정확도: {model.evaluate(binary_2gram_test_ds)[1]:.3f}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vJTgC5qDPHWL"
      },
      "source": [
        "#### TF-IDF 인코딩을 사용한 바이그램"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R6zaQL_wPHWL"
      },
      "source": [
        "**토큰 카운트를 반환하는 `TextVectorization` 층**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "TAHAvzsgPHWL"
      },
      "outputs": [],
      "source": [
        "text_vectorization = TextVectorization(\n",
        "    ngrams=2,\n",
        "    max_tokens=20000,\n",
        "    output_mode=\"count\"\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ukzyV24GPHWL"
      },
      "source": [
        "**TF-IDF 가중치가 적용된 출력을 반환하는 `TextVectorization` 층**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "6Um9A6ydPHWL"
      },
      "outputs": [],
      "source": [
        "text_vectorization = TextVectorization(\n",
        "    ngrams=2,\n",
        "    max_tokens=20000,\n",
        "    output_mode=\"tf_idf\",\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BwoBv2nqPHWM"
      },
      "source": [
        "**TF-IDF 바이그램 모델 훈련하고 테스트하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "AGhSasurPHWM",
        "outputId": "8110eafb-226a-4b5c-f058-4d79f0977f05",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_4\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_5 (InputLayer)        [(None, 20000)]           0         \n",
            "                                                                 \n",
            " dense_8 (Dense)             (None, 16)                320016    \n",
            "                                                                 \n",
            " dropout_4 (Dropout)         (None, 16)                0         \n",
            "                                                                 \n",
            " dense_9 (Dense)             (None, 1)                 17        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 320033 (1.22 MB)\n",
            "Trainable params: 320033 (1.22 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n",
            "Epoch 1/10\n",
            "625/625 [==============================] - 6s 9ms/step - loss: 0.5076 - accuracy: 0.7673 - val_loss: 0.2983 - val_accuracy: 0.8894\n",
            "Epoch 2/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.3358 - accuracy: 0.8559 - val_loss: 0.3017 - val_accuracy: 0.8704\n",
            "Epoch 3/10\n",
            "625/625 [==============================] - 4s 6ms/step - loss: 0.2955 - accuracy: 0.8723 - val_loss: 0.3197 - val_accuracy: 0.8888\n",
            "Epoch 4/10\n",
            "625/625 [==============================] - 4s 6ms/step - loss: 0.2782 - accuracy: 0.8812 - val_loss: 0.3548 - val_accuracy: 0.8856\n",
            "Epoch 5/10\n",
            "625/625 [==============================] - 3s 5ms/step - loss: 0.2561 - accuracy: 0.8917 - val_loss: 0.3496 - val_accuracy: 0.8840\n",
            "Epoch 6/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.2462 - accuracy: 0.9006 - val_loss: 0.3562 - val_accuracy: 0.8690\n",
            "Epoch 7/10\n",
            "625/625 [==============================] - 3s 5ms/step - loss: 0.2369 - accuracy: 0.8967 - val_loss: 0.3624 - val_accuracy: 0.8896\n",
            "Epoch 8/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.2291 - accuracy: 0.9051 - val_loss: 0.3716 - val_accuracy: 0.8770\n",
            "Epoch 9/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.2244 - accuracy: 0.9062 - val_loss: 0.3864 - val_accuracy: 0.8772\n",
            "Epoch 10/10\n",
            "625/625 [==============================] - 3s 4ms/step - loss: 0.2196 - accuracy: 0.9071 - val_loss: 0.4029 - val_accuracy: 0.8814\n",
            "782/782 [==============================] - 6s 7ms/step - loss: 0.3042 - accuracy: 0.8860\n",
            "테스트 정확도: 0.886\n"
          ]
        }
      ],
      "source": [
        "# 텐서플로 2.8.x 버전에서 TF-IDF 인코딩을 GPU에서 수행할 때 오류가 발생할 수 있습니다.\n",
        "# 텐서플로 2.9에서 이 이슈가 해결되었지만 코드를 테스트할 시점에 코랩의 텐서플로 버전은 2.8.2이기 때문에\n",
        "# 에러를 피하기 위해 CPU를 사용하여 텍스트를 변환합니다.\n",
        "with tf.device(\"cpu\"):\n",
        "    text_vectorization.adapt(text_only_train_ds)\n",
        "\n",
        "tfidf_2gram_train_ds = train_ds.map(\n",
        "    lambda x, y: (text_vectorization(x), y),\n",
        "    num_parallel_calls=4)\n",
        "tfidf_2gram_val_ds = val_ds.map(\n",
        "    lambda x, y: (text_vectorization(x), y),\n",
        "    num_parallel_calls=4)\n",
        "tfidf_2gram_test_ds = test_ds.map(\n",
        "    lambda x, y: (text_vectorization(x), y),\n",
        "    num_parallel_calls=4)\n",
        "\n",
        "model = get_model()\n",
        "model.summary()\n",
        "callbacks = [\n",
        "    keras.callbacks.ModelCheckpoint(\"tfidf_2gram.h5\",\n",
        "                                    save_best_only=True)\n",
        "]\n",
        "model.fit(tfidf_2gram_train_ds.cache(),\n",
        "          validation_data=tfidf_2gram_val_ds.cache(),\n",
        "          epochs=10,\n",
        "          callbacks=callbacks)\n",
        "model = keras.models.load_model(\"tfidf_2gram.h5\")\n",
        "print(f\"테스트 정확도: {model.evaluate(tfidf_2gram_test_ds)[1]:.3f}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "tGIbe1KrPHWM"
      },
      "outputs": [],
      "source": [
        "inputs = keras.Input(shape=(1,), dtype=\"string\")\n",
        "processed_inputs = text_vectorization(inputs)\n",
        "outputs = model(processed_inputs)\n",
        "inference_model = keras.Model(inputs, outputs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "XBEraM_0PHWM",
        "outputId": "1fb411da-3d03-4ebf-a4de-76364f4e5940",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "긍정적인 리뷰일 확률: 93.58 퍼센트\n"
          ]
        }
      ],
      "source": [
        "import tensorflow as tf\n",
        "raw_text_data = tf.convert_to_tensor([\n",
        "    [\"That was an excellent movie, I loved it.\"],\n",
        "])\n",
        "predictions = inference_model(raw_text_data)\n",
        "print(f\"긍정적인 리뷰일 확률: {float(predictions[0] * 100):.2f} 퍼센트\")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "chapter11_part01_introduction.i",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "default:Python",
      "language": "python",
      "name": "conda-env-default-py"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.10"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}