02 - 03 - LSTM will find the author.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# We can use Long Short Term Memory (LSTM) to find the author of the text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "import tensorflow_datasets as tfds\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIRECTORY_URL = 'https://storage.googleapis.com/download.tensorflow.org/data/illiad/'\n",
    "FILE_NAMES = ['cowper.txt', 'derby.txt', 'butler.txt']\n",
    "\n",
    "for name in FILE_NAMES:\n",
    "    text_dir = tf.keras.utils.get_file(name, origin=DIRECTORY_URL + name)\n",
    "\n",
    "parent_dir = os.path.dirname(text_dir)\n",
    "\n",
    "parent_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!curl https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt > $HOME/.keras/datasets/shakespeare.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "FILE_NAMES = ['cowper.txt', 'derby.txt', 'butler.txt', 'shakespeare.txt']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def labeler(example, index):\n",
    "    return example, tf.cast(index, tf.int64)\n",
    "\n",
    "\n",
    "labeled_data_sets = []\n",
    "\n",
    "for i, file_name in enumerate(FILE_NAMES):\n",
    "    lines_dataset = tf.data.TextLineDataset(os.path.join(\n",
    "        parent_dir, file_name))\n",
    "    labeled_dataset = lines_dataset.map(lambda ex: labeler(ex, i))\n",
    "    labeled_data_sets.append(labeled_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BUFFER_SIZE = 50000\n",
    "BATCH_SIZE = 64\n",
    "TAKE_SIZE = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_labeled_data = labeled_data_sets[0]\n",
    "for labeled_dataset in labeled_data_sets[1:]:\n",
    "    all_labeled_data = all_labeled_data.concatenate(labeled_dataset)\n",
    "\n",
    "all_labeled_data = all_labeled_data.shuffle(BUFFER_SIZE,\n",
    "                                            reshuffle_each_iteration=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ex in all_labeled_data.take(15):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = tfds.features.text.Tokenizer()\n",
    "\n",
    "vocabulary_set = set()\n",
    "for text_tensor, _ in all_labeled_data:\n",
    "    some_tokens = tokenizer.tokenize(text_tensor.numpy())\n",
    "    vocabulary_set.update(some_tokens)\n",
    "\n",
    "vocab_size = len(vocabulary_set)\n",
    "vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = tfds.features.text.TokenTextEncoder(vocabulary_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "example_text = next(iter(all_labeled_data))[0].numpy()\n",
    "print(example_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded_example = encoder.encode(example_text)\n",
    "print(encoded_example)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode(text_tensor, label):\n",
    "    encoded_text = encoder.encode(text_tensor.numpy())\n",
    "    return encoded_text, label\n",
    "\n",
    "\n",
    "def encode_map_fn(text, label):\n",
    "    return tf.py_function(encode, inp=[text, label], Tout=(tf.int64, tf.int64))\n",
    "\n",
    "\n",
    "all_encoded_data = all_labeled_data.map(encode_map_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = all_encoded_data.skip(TAKE_SIZE).shuffle(BUFFER_SIZE)\n",
    "train_data = train_data.padded_batch(BATCH_SIZE, padded_shapes=([-1],[]))\n",
    "\n",
    "test_data = all_encoded_data.take(TAKE_SIZE)\n",
    "test_data = test_data.padded_batch(BATCH_SIZE, padded_shapes=([-1],[]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_text, sample_labels = next(iter(test_data))\n",
    "\n",
    "sample_text[0], sample_labels[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.Sequential()\n",
    "model.add(tf.keras.layers.Embedding(vocab_size, 64))\n",
    "model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)))\n",
    "for units in [64, 64]:\n",
    "    model.add(tf.keras.layers.Dense(units, activation='relu'))\n",
    "\n",
    "# Output layer. The first argument is the number of labels.\n",
    "model.add(tf.keras.layers.Dense(4, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='adam',\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# ~4 minutes\n",
    "model.fit(train_data, epochs=3, validation_data=test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_loss, eval_acc = model.evaluate(test_data)\n",
    "\n",
    "print('\\nEval loss: {:.3f}, Eval accuracy: {:.3f}'.format(eval_loss, eval_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "example_text = b'Do not omit the heavy offer of it'\n",
    "encoded_example = encoder.encode(example_text)\n",
    "print(encoded_example)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.predict( [encoded_example,] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}