memorizeLM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model the properties of text\n",
    "This model will learn to predict what will follow in the sequence. Such a model is called a language model.\n",
    "\n",
    "Most language models use words as units of the sequence. We will use characters. There are two reasons for this:\n",
    "\n",
    "- a word-based model ignores the whitespace between words (may be  relevant   for    poetry)\n",
    "- there are fewer different characters than words and there are more characters than words in every line of poetry, which simplifies our task given little training data\n",
    "\n",
    "Language models can be used for _generation_, i.e., we can sample texts from the model that are similar to the training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import statements for modules that we may need below.\n",
    "import dynet as dy\n",
    "import sys\n",
    "from random import shuffle, random\n",
    "\n",
    "# read in the words and set up the \"input vocabulary\" (in this case: all characters)\n",
    "data = []\n",
    "# we'll use only one style of lines -- sound-poetry can be hard to distinguish from random results...\n",
    "with open('parlando.lines') as f:\n",
    "    data.extend([l.strip() for l in f.readlines()])\n",
    "\n",
    "characters = set(\"\".join(data))\n",
    "characters.add(\"<EOS>\") # special tag for end of sentence\n",
    "\n",
    "int2char = list(characters)\n",
    "char2int = {c:i for i,c in enumerate(characters)}\n",
    "\n",
    "VOCAB_SIZE = len(characters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# return compute loss of RNN for one sentence\n",
    "def do_one_sentence(rnn, params, line):\n",
    "    # setup the sentence\n",
    "    dy.renew_cg()\n",
    "    s0 = rnn.initial_state()\n",
    "    R = dy.parameter(params[\"R\"])\n",
    "    bias = dy.parameter(params[\"bias\"])\n",
    "    lookup = params[\"lookup\"]\n",
    "    line = [\"<EOS>\"] + list(line) + [\"<EOS>\"]\n",
    "    line = [char2int[c] for c in line]\n",
    "    s = s0\n",
    "    loss = []\n",
    "    for char,next_char in zip(line,line[1:]):\n",
    "        s = s.add_input(lookup[char])\n",
    "        probs = R*s.output() + bias\n",
    "        loss.append(dy.pickneglogsoftmax(probs,next_char))\n",
    "    loss = dy.esum(loss)\n",
    "    return loss\n",
    "\n",
    "# generate from model:\n",
    "def generate(rnn, params):\n",
    "    def sample(probs):\n",
    "        rnd = random()\n",
    "        for i,p in enumerate(probs):\n",
    "            rnd -= p\n",
    "            if rnd <= 0: break\n",
    "        return i\n",
    "    # setup the sentence\n",
    "    dy.renew_cg()\n",
    "    s0 = rnn.initial_state()\n",
    "    \n",
    "    R = dy.parameter(params[\"R\"])\n",
    "    bias = dy.parameter(params[\"bias\"])\n",
    "    lookup = params[\"lookup\"]\n",
    "    \n",
    "    s = s0.add_input(lookup[char2int[\"<EOS>\"]])\n",
    "    out=[]\n",
    "    while True:\n",
    "        probs = dy.softmax(R*s.output() + bias)\n",
    "        probs = probs.vec_value()\n",
    "        next_char = sample(probs)\n",
    "        out.append(int2char[next_char])\n",
    "        if out[-1] == \"<EOS>\": break\n",
    "        s = s.add_input(lookup[next_char])\n",
    "    return \"\".join(out[:-1]) # strip the <EOS>\n",
    "\n",
    "# train, and generate every 5 samples\n",
    "def train(rnn, params, lines):\n",
    "    trainer = trainer_type(pc)\n",
    "    for i in range(ITERATIONS):\n",
    "        for line in lines:\n",
    "            loss = do_one_sentence(rnn, params, line)\n",
    "            loss_value = loss.value()\n",
    "            loss.backward()\n",
    "            trainer.update()\n",
    "        if i % 5 == 0: \n",
    "            print(\"%.10f\" % loss_value, end=\"\\t\")\n",
    "            print(generate(rnn, params))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ITERATIONS = 200\n",
    "\n",
    "INPUT_DIM = 40\n",
    "HIDDEN_DIM = 50\n",
    "LAYERS = 1\n",
    "\n",
    "builder_type = dy.SimpleRNNBuilder\n",
    "#builder_type = dy.LSTMBuilder\n",
    "\n",
    "pc = dy.ParameterCollection()\n",
    "rnn = builder_type(LAYERS, INPUT_DIM, HIDDEN_DIM, pc)\n",
    "# add parameters for the hidden->output part for both lstm and srnn\n",
    "params = {}\n",
    "params[\"lookup\"] = pc.add_lookup_parameters((VOCAB_SIZE, INPUT_DIM))\n",
    "params[\"R\"] = pc.add_parameters((VOCAB_SIZE, HIDDEN_DIM))\n",
    "params[\"bias\"] = pc.add_parameters((VOCAB_SIZE))\n",
    "\n",
    "trainer_type = dy.SimpleSGDTrainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sentence = \"blaukraut bleibt blaukraut und brautkleid bleibt brautkleid\"\n",
    "train(rnn, params, [sentence])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train(rnn, params, data)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}