/
ModelTest.py
129 lines (114 loc) · 5.67 KB
/
ModelTest.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
#!/usr/bin/env python
# -*- coding: utf-8 -*-
__author__ = 'Yunchuan Chen'
from models import SimpleLangModel, NCELangModel, NCELangModelV1, TreeLangModel
from keras.layers.core import Dropout, Dense
import os
import logging
import optparse
import cPickle as pickle
import theano
floatX = theano.config.floatX
parser = optparse.OptionParser(usage="%prog [OPTIONS]")
parser.add_option("-q", "--quiet",
action="store_false", dest="verbose", default=True,
help="don't print progress bar to stdout")
parser.add_option("-s", "--simple",
action="store_true", dest="train_simple", default=False,
help="Train Simple language model")
parser.add_option("-n", "--nce",
action="store_true", dest="train_nce", default=False,
help="Train NCE based language model")
parser.add_option("-c", "--nce1",
action="store_true", dest="train_nce1", default=False,
help="Train NCE based language model V1")
parser.add_option("-b", "--batch-size", type='int', dest="batch_size", default=256,
help="Batch size")
parser.add_option("-t", "--test",
action="store_true", dest="test", default=False,
help="train on small data set")
parser.add_option("-r", "--tree",
action="store_true", dest="tree", default=False,
help="Train hierarchical softmax language model")
parser.add_option("-m", "--tree-type", type='str',
dest="tree_type", default='huffman',
help="Specify the type of the tree")
parser.add_option("-d", "--debug",
action="store_true", dest="debug", default=False,
help="show debug information")
parser.add_option("-g", "--unigram",
action="store_true", dest="unigram", default=False,
help="Whether use unigram distribution for noise samples")
parser.add_option("-z", "--optimizer", type='str',
dest="optimizer", default='adam',
help="Specify optimizer")
parser.add_option('-a', "--attention", dest="attention", type='str', default='none',
help='Specify attention model')
parser.add_option('-l', '--attention-length', dest='att_len', type='int', default=10,
help='Specify attention bias length')
options, args = parser.parse_args()
# ====================================================================================
# if TESTLM environment variable is defined, run the program on a small data set.
if os.environ.get('TESTLM') is not None or options.test:
data_path = os.path.abspath('data/fake/test')
else:
data_path = os.path.abspath('data/fake')
#data_path = os.path.abspath('data/fake/test')
if options.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
if options.unigram:
import cPickle as pickle
with file(os.path.join(data_path, 'meta.pkl'), 'rb') as mf:
meta = pickle.load(mf)
negprob_table = meta['rel_freq']
else:
negprob_table = None
if options.train_simple:
logging.info('Train simple language model')
model = SimpleLangModel(vocab_size=15, embed_dims=128, context_dims=128, optimizer=options.optimizer)
model.compile()
model.train_from_dir(data_path, validation_split=0.05, batch_size=options.batch_size, verbose=options.verbose)
if options.train_nce:
logging.info('Train NCE based language model')
model = NCELangModel(vocab_size=15, nb_negative=2, embed_dims=128, negprob_table=negprob_table,
optimizer=options.optimizer)
model.compile()
logging.debug('compile success')
model.train_from_dir(data_path, validation_split=0.05, batch_size=options.batch_size, verbose=options.verbose)
if options.train_nce1:
logging.info('Train NCE based language model (1)')
model = NCELangModelV1(vocab_size=15, nb_negative=6, embed_dims=128, negprob_table=negprob_table,
optimizer=options.optimizer)
model.compile()
logging.debug('compile success')
model.train_from_dir(data_path, validation_split=0.05, batch_size=options.batch_size, verbose=options.verbose)
if options.tree:
logging.info('Train hierarchical softmax language model')
if options.tree_type == 'huffman':
logging.info('train with Huffman Tree')
tree_file = 'data/fake/tree-info-huffman.pkl'
else:
tree_file = 'data/fake/tree-info.pkl'
logging.info('Train with Brown tree')
with file('data/fake/tree-info.pkl', 'rb') as f:
tree_info = pickle.load(f)
word2cls = tree_info['idx2cls']
word2bitstr = tree_info['idx2bitstr'].astype(floatX)
model = TreeLangModel(vocab_size=15, embed_dim=128, cntx_dim=128, word2class=word2cls, word2bitstr=word2bitstr)
model.compile(optimizer=options.optimizer)
logging.debug('compile success')
model.train_from_dir(data_path, validation_split=0.05, batch_size=options.batch_size, verbose=options.verbose)
if options.attention == 'simple':
from models import SimpAttLangModel
model = SimpAttLangModel(vocab_size=15, embed_dims=128, context_dim=128, attention_len=options.att_len)
model.compile()
logging.debug('compile success')
model.train_from_dir(data_path, validation_split=0.05, batch_size=options.batch_size, verbose=options.verbose)
if options.attention == 'parallel':
from models import ParallelAttLangModel
model = ParallelAttLangModel(vocab_size=15, embed_dims=128, context_dim=128, attention_len=options.att_len)
model.compile()
logging.debug('compile success')
model.train_from_dir(data_path, validation_split=0.05, batch_size=options.batch_size, verbose=options.verbose)