keras_example_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "import io\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras as keras\n",
    "from tensorflow.keras import layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.3.1\n"
     ]
    }
   ],
   "source": [
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus length: 3204159\n",
      "Total characters: 103\n",
      "Number of sequences: 1068040\n"
     ]
    }
   ],
   "source": [
    "path = \"infinite_jest_text.txt\"\n",
    "\n",
    "with io.open(path, encoding=\"utf-8\") as f:\n",
    "    text = f.read().lower()\n",
    "    \n",
    "text = text.replace(\"\\n\", \" \")\n",
    "print(\"Corpus length: {}\".format(len(text)))\n",
    "\n",
    "chars = sorted(list(set(text)))\n",
    "print(\"Total characters: {}\".format(len(chars)))\n",
    "\n",
    "char_indices = dict((c, i) for i, c in enumerate(chars))\n",
    "indices_char = dict((i, c) for i, c in enumerate(chars))\n",
    "\n",
    "# cut the text in semi-redundant sequences of maxlen characters\n",
    "maxlen = 40\n",
    "step = 3\n",
    "sentences = []\n",
    "next_chars = []\n",
    "\n",
    "# for every 40 characters create an example + its next char\n",
    "for i in range(0, len(text) - maxlen, step):\n",
    "    sentences.append(text[i:i+maxlen]) # this doesnt include i+mxlen\n",
    "    next_chars.append(text[i + maxlen]) # this is i+mxlen\n",
    "print(\"Number of sequences: {}\".format(len(sentences)))\n",
    "\n",
    "x = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)\n",
    "y = np.zeros((len(sentences), len(chars)), dtype=np.bool)\n",
    "\n",
    "# for each character in each \"sentence\", encode it (as positive) in the \n",
    "# example. Set the label to be the next character after that \"sentence\"\n",
    "for i, sentence in enumerate(sentences):\n",
    "    for t, char in enumerate(sentence):\n",
    "        x[i, t, char_indices[char]] = 1\n",
    "    y[i, char_indices[next_chars[i]]] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.Sequential(\n",
    "    [\n",
    "        keras.Input(shape=(maxlen, len(chars))),\n",
    "        layers.LSTM(64),\n",
    "        layers.Dense(len(chars), activation=\"softmax\")\n",
    "    ])\n",
    "\n",
    "optimiser = keras.optimizers.Adam()\n",
    "model.compile(loss=\"categorical_crossentropy\", optimizer=optimiser, metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample(preds, temperature=1.0):\n",
    "    \"\"\"Helper function to sample an index from a probability array.\"\"\"\n",
    "    preds = np.asarray(preds).astype(\"float64\")\n",
    "    preds = np.log(preds) / temperature\n",
    "    exp_preds = np.exp(preds)\n",
    "    preds = exp_preds / np.sum(exp_preds) # normalise\n",
    "    probas = np.random.multinomial(1, preds, 1)\n",
    "    return np.argmax(probas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 120\n",
    "batch_size = 128\n",
    "\n",
    "with open(\"output.txt\", \"w\") as out:\n",
    "    for epoch in range(epochs):\n",
    "        model.fit(x, y, batch_size=batch_size, epochs=1)\n",
    "\n",
    "\n",
    "        if epoch % 10 == 0:\n",
    "            out.write(\"\\n\")\n",
    "            out.write(\"Generating text after epoch: {0:d}\\n\".format(epoch))\n",
    "\n",
    "            start_index = random.randint(0, len(text)-maxlen-1)\n",
    "            for diversity in [0.2, 0.5, 1.0, 1.2]:\n",
    "                out.write(\"...Diversity: {}\\n\".format(diversity))\n",
    "\n",
    "                generated = \"\"\n",
    "                sentence = text[start_index : start_index+maxlen]\n",
    "                out.write(\"...Generating with seed: '{}'\\n\".format(sentence))\n",
    "\n",
    "                for i in range(400):\n",
    "                    x_pred = np.zeros((1, maxlen, len(chars)))\n",
    "                    for t, char in enumerate(sentence):\n",
    "                        x_pred[0, t, char_indices[char]] = 1.0\n",
    "                    preds = model.predict(x_pred, verbose=0)[0]\n",
    "                    next_index = sample(preds, diversity)\n",
    "                    next_char = indices_char[next_index]\n",
    "                    sentence = sentence[1:] + next_char\n",
    "                    generated += next_char\n",
    "\n",
    "                out.write(\"...Generated: {}\\n\\n\".format(generated))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_sequences(model, text, indices_char):   \n",
    "    start_index = random.randint(0, len(text)-maxlen-1)\n",
    "    for diversity in [0.2, 0.5, 1.0, 1.2]:\n",
    "        print(\"...Diversity: {}\".format(diversity))\n",
    "        \n",
    "        generated = \"\"\n",
    "        sentence = text[start_index : start_index+maxlen]\n",
    "        print(\"...Generating with seed: '{}'\".format(sentence))\n",
    "        \n",
    "        for i in range(400):\n",
    "            x_pred = np.zeros((1, maxlen, len(chars)))\n",
    "            for t, char in enumerate(sentence):\n",
    "                x_pred[0, t, char_indices[char]] = 1.0\n",
    "            preds = model.predict(x_pred, verbose=0)[0]\n",
    "            next_index = sample(preds, diversity)\n",
    "            next_char = indices_char[next_index]\n",
    "            sentence = sentence[1:] + next_char\n",
    "            generated += next_char\n",
    "            \n",
    "        print(\"...Generated: \".format(generated))\n",
    "        print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}