04-Cornell-Movie-Dialog-Bot.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demonstrate Seq2Seq Wrapper with Cornell Movie Dialog Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "# preprocessed data\n",
    "from datasets.cornell_corpus import data\n",
    "import data_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'datasets.cornell_corpus.data' from '/home/suriya/_/tf/tf-seq2seq-wrapper/datasets/cornell_corpus/data.py'>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import importlib\n",
    "importlib.reload(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load data from pickle and npy files\n",
    "metadata, idx_q, idx_a = data.load_data(PATH='datasets/cornell_corpus/')\n",
    "(trainX, trainY), (testX, testY), (validX, validY) = data_utils.split_dataset(idx_q, idx_a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# parameters \n",
    "xseq_len = trainX.shape[-1]\n",
    "yseq_len = trainY.shape[-1]\n",
    "batch_size = 16\n",
    "xvocab_size = len(metadata['idx2w'])  \n",
    "yvocab_size = xvocab_size\n",
    "emb_dim = 1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import seq2seq_wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<log> Building Graph </log>"
     ]
    }
   ],
   "source": [
    "model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len,\n",
    "                               yseq_len=yseq_len,\n",
    "                               xvocab_size=xvocab_size,\n",
    "                               yvocab_size=yvocab_size,\n",
    "                               ckpt_path='ckpt/cornell_corpus/',\n",
    "                               emb_dim=emb_dim,\n",
    "                               num_layers=3\n",
    "                               )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_batch_gen = data_utils.rand_batch_gen(validX, validY, 32)\n",
    "test_batch_gen = data_utils.rand_batch_gen(testX, testY, 256)\n",
    "train_batch_gen = data_utils.rand_batch_gen(trainX, trainY, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = model.restore_last_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(256, 25)\n"
     ]
    }
   ],
   "source": [
    "input_ = test_batch_gen.__next__()[0]\n",
    "output = model.predict(sess, input_)\n",
    "print(output.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "q : [bill id like you to meet jack torrance]; a : [how much do you think]\n",
      "q : [and who are you to talk you were nothing you couldnt even sing i must have been out of my mind]; a : [i dont know what youre talking about]\n",
      "q : [by breaking up a companys assets]; a : [what are you talking about]\n",
      "q : [what is it]; a : [i dont know]\n",
      "q : [ill see you there]; a : [ill get out]\n",
      "q : [okay ill be talking to you]; a : [youre not going to get out]\n",
      "q : [i must be outta my mind buddy quit it]; a : [okay for a minute]\n",
      "q : [when are you going to let the police know]; a : [you dont know what youre talking about]\n",
      "q : [you can do it]; a : [yeah i think so]\n",
      "q : [like hell you know if you fellows stuck together stead of letting them walk all over you they might not try it]; a : [if you werent talking about me i dont know what youre talking about]\n",
      "q : [wait are you saying that i dont appreciate]; a : [i know you know what i mean]\n",
      "q : [no you just looked as if you did]; a : [well i didnt believe you]\n",
      "q : [youre terrible]; a : [yeah but im not]\n",
      "q : [you would take a pain n the arse full stop]; a : [dont worry about it]\n",
      "q : [yeah howd the kid hold up]; a : [whats that]\n",
      "q : [yes well miss ratched feels that youre a disturbing influence on the other patients]; a : [dont you tell me]\n",
      "q : [you leading a squad]; a : [what is it]\n",
      "q : [nah i dont think so]; a : [why not]\n",
      "q : [hes pissing in our faces again and were just taking it]; a : [i cant]\n",
      "q : [my lady]; a : [she said you were a little girl]\n",
      "q : [im sorry i dont know what else to say except im sorry]; a : [why would you like to be]\n",
      "q : [mr brandon couldnt be here he might have left something for you what would it look like]; a : [thats it i think i can get it out]\n",
      "q : [nobodys gonna get you now get inside]; a : [i cant go back]\n",
      "q : [i just turned 25 i was 24 for a whole year]; a : [well i dont think so]\n",
      "q : [make sure he doesnt leave]; a : [you sure]\n",
      "q : [do we need him]; a : [i dont know what to do]\n",
      "q : [youre terry unk arent you]; a : [what are you doing here]\n",
      "q : [so lets go]; a : [i dont think so]\n",
      "q : [ya owe me twentyfive bucks]; a : [its a good man]\n",
      "q : [shall i leave]; a : [of course]\n",
      "q : [unk unk probably asleep by now do you want him to see you like this]; a : [no way i dont want to get out of here]\n",
      "q : [well i really think hes got a chance]; a : [i know]\n",
      "q : [youd better be quiet sandy]; a : [shut up]\n",
      "q : [buddy that was pretty unk of you pushing me away like that just when it was interesting]; a : [what about you]\n",
      "q : [jesus christ you scared the shit out of me]; a : [whats going on]\n",
      "q : [well im sorry im really sorry ellie]; a : [its okay]\n",
      "q : [youre not a man because of a job duff]; a : [but i said that]\n",
      "q : [he didnt lose her he threw her away]; a : [hes not dead]\n",
      "q : [you unk have the gotta]; a : [what the hell are you talking about]\n",
      "q : [whoa whoa what do you expect them to say youre alan unk]; a : [im sorry i dont know]\n",
      "q : [doc what can i tell ya]; a : [go ahead]\n",
      "q : [my lady this play will end badly i will tell]; a : [lets get out of here]\n",
      "q : [what if i said goodbye]; a : [then why are you]\n",
      "q : [im going to miss you]; a : [no youre not]\n",
      "q : [well wed love to but were going to another party]; a : [what do you mean]\n",
      "q : [dog eat dog unk you fuck other man before he fuck you and you must fuck last]; a : [what do you think we can do]\n",
      "q : [not in the trunk]; a : [what about]\n",
      "q : [can we light the candles now on the cake]; a : [you want to go home]\n",
      "q : [you look frightened have i been saying something frightening]; a : [i dont know what you mean]\n",
      "q : [you want to hear the good news first or the bad news]; a : [what the hell is that]\n",
      "q : [ya gotta be a little soft to wanna be a unk its a racket where ya almost guaranteed to end up a bum]; a : [all right]\n",
      "q : [you werent going with her]; a : [well shes not a little girl]\n",
      "q : [i dont want land]; a : [you dont have to be a good man]\n",
      "q : [what do you want from me cotton]; a : [we dont know what youre talking about]\n",
      "q : [the hopes are perfect beautiful identical smooth and they are for something really amazing i feel it in my bones]; a : [youre not a fool]\n",
      "q : [get married]; a : [what do you think]\n",
      "q : [ha dear boy i do hope this doesnt unk a meeting in private]; a : [ill get out of here]\n",
      "q : [karl you up]; a : [yeah im not going to get out of here]\n",
      "q : [i just realized this is for television isnt it i cant swear up and down like i just did]; a : [its not that you know what youre talking about]\n",
      "q : [yes george]; a : [and what do you want]\n",
      "q : [hey eddie looks like you really stepped in it this time]; a : [thats right i dont know]\n",
      "q : [you dont want to go all the way to san francisco in a unk do you i dont]; a : [im not sure]\n",
      "q : [it can happen so sudden cant it being left out on your own]; a : [its not a good idea]\n",
      "q : [what do you mean]; a : [i dont know i dont know what i mean]\n",
      "q : [i want to]; a : [you want to go]\n",
      "q : [what do you want take my wife please]; a : [well i dont know what to do]\n",
      "q : [look at this the lock is totally unk]; a : [you know what it is]\n",
      "q : [he wont steal im tellin you hes a pretty good ol boy keeps to himself]; a : [thats right]\n",
      "q : [youre really pushing it bringing me here]; a : [what do you want me to do]\n",
      "q : [im not unk with armitage and his unk breathing down my neck]; a : [you mean you dont want to go]\n",
      "q : [my god these people are insane]; a : [we dont know what they are]\n",
      "q : [no it wasnt like that]; a : [yes you did]\n",
      "q : [dont you worry about that]; a : [dont worry about me]\n",
      "q : [well i just kept unk that dish maybe it doesnt sound very sexy but it was]; a : [yeah i dont know]\n",
      "q : [hey vaughan how are you karl]; a : [im fine]\n",
      "q : [this isnt your room youre in unk i fucked up]; a : [im sorry i dont know what to do]\n",
      "q : [r was worried about you you didnt even call youre always on my case if i dont call]; a : [i know you dont know what youre talking about]\n",
      "q : [something happened you got unk in the last quarter]; a : [what do you want to do]\n",
      "q : [if youre going to be unk i wish youd be a little more discreet about it rich men like unk love you and leave you]; a : [thats not what i mean]\n",
      "q : [how long would they let me sleep]; a : [i dont know i dont know what to do]\n",
      "q : [whats got billy so unk]; a : [i dont know what to do with it]\n",
      "q : [you fucking bastard]; a : [come on]\n",
      "q : [she used to call me mr right remember that buddy]; a : [how are you going to do]\n",
      "q : [my unk in miami its nice down there]; a : [thats what i said]\n",
      "q : [speak with my lawyer]; a : [you dont know what you mean]\n",
      "q : [mr unk because you love the theatre you must have a part in my play i am writing an unk a small but vital role]; a : [thats right i dont think so]\n",
      "q : [of course you dont know anything about it if you knew anything about it i wouldnt have to send you over there to cover it]; a : [thats not what you want]\n",
      "q : [does mom know]; a : [she said she was in love]\n",
      "q : [its okay tatum shes just doing her job right gale]; a : [shes not going to see her]\n",
      "q : [wanna keep goin]; a : [im sorry]\n",
      "q : [i thought so too unk a neat guy]; a : [hey i dont know what youre doing to do with it]\n",
      "q : [did i wake you]; a : [no you dont]\n",
      "q : [how could you do this]; a : [well i dont know what i mean]\n",
      "q : [that can wait till the weekend]; a : [are you kidding]\n",
      "q : [if you cant look anymore i understand]; a : [im not going to tell you that]\n",
      "q : [maybe i can change him]; a : [then he doesnt know what he is]\n",
      "q : [maybe if i kiss him ill feel it]; a : [then you should have to]\n",
      "q : [well youre a little early]; a : [i dont know what youre doing to get out of here]\n",
      "q : [i thought michael was picking me up]; a : [dont worry about it im not going to get out of here]\n",
      "q : [i dont believe he did sir i couldnt find a single track just doesnt make sense]; a : [shut up what do you mean]\n",
      "q : [he said that]; a : [he was here]\n",
      "q : [its unk unk the name of the character she plays in the movie]; a : [i know what i mean]\n",
      "q : [hes company]; a : [hes not going to see him]\n",
      "q : [youre so proud youre like some retarded kid comin home from school look dad i got an f]; a : [thats right thats what you want to do]\n",
      "q : [feels like theres a bullet still in my chest]; a : [right for me]\n",
      "q : [not now charlie ive got a headache get used to the word roll it around your tongue for a years]; a : [its the time i dont know what to do]\n",
      "q : [and the bookstore have you been working there long]; a : [i think so]\n",
      "q : [you sent for me]; a : [yes yes i am]\n",
      "q : [damour damour why do i know that name]; a : [well i dont know]\n",
      "q : [speaking of which you run that license plate for me]; a : [i cant believe you]\n",
      "q : [i dont know unk a unk]; a : [you know what i mean]\n",
      "q : [how shes even forgot her own language]; a : [long night she was a very nice person]\n",
      "q : [just that she got away]; a : [well i think so]\n",
      "q : [anything i dont care what it is just so its something]; a : [its not that i dont know]\n",
      "q : [at this hour]; a : [no problem]\n",
      "q : [i swear it he wants romeo for ned and the unk men]; a : [and how much do you think]\n",
      "q : [oh really i thought it was pretty good]; a : [its a good idea]\n",
      "q : [look fry company says were responsible for every one of those]; a : [dont be silly]\n",
      "q : [laser unk you cant get the code wrong it unk you i cant let you try it]; a : [ill take care of you]\n",
      "q : [what if we were to put bruce into the park as a guest]; a : [dont worry ill be fine if you want to get out of here]\n",
      "q : [could be]; a : [but what do you think]\n",
      "q : [i dont know i cant say]; a : [then what did you do]\n"
     ]
    }
   ],
   "source": [
    "replies = []\n",
    "for ii, oi in zip(input_.T, output):\n",
    "    q = data_utils.decode(sequence=ii, lookup=metadata['idx2w'], separator=' ')\n",
    "    decoded = data_utils.decode(sequence=oi, lookup=metadata['idx2w'], separator=' ').split(' ')\n",
    "    if decoded.count('unk') == 0:\n",
    "        if decoded not in replies:\n",
    "            print('q : [{0}]; a : [{1}]'.format(q, ' '.join(decoded)))\n",
    "            replies.append(decoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}