-
Notifications
You must be signed in to change notification settings - Fork 2
/
experiments_other.py
77 lines (62 loc) · 3.71 KB
/
experiments_other.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
#from dota_test import run_once
from uci_test import run_once as run_once_poker
from datetime import datetime
import numpy as np
#N.B: LR for these results was tuned incrorectly, it should be 0.001 for all results
hparams = [
#{ 'model_type': 'set-nn-max', 'dataset': 'teamfights', 'lr': 0.0005,
#'mem_dim': 64, 'batch_size': 4, 'epochs': 1 },
#{'model_type': 'json-nn', 'dataset': 'teamfights', 'lr': 0.0005,
#'mem_dim': 128, 'batch_size': 4, 'epochs': 1 },
#{'model_type': 'json-nn', 'dataset': 'poker', 'lr': 0.0005, #55.11
#'mem_dim': 128, 'batch_size': 64, 'epochs': 50, 'poker_frac_test': 0.05 },
#{'model_type': 'json-nn', 'dataset': 'poker', 'lr': 0.00025, #91.6
#'mem_dim': 128, 'batch_size': 4, 'epochs': 100, 'poker_frac_test': 0.2 },
#{'model_type': 'json-nn', 'dataset': 'poker', 'lr': 0.0005, #98.60
#'mem_dim': 128, 'batch_size': 16, 'epochs': 50, 'poker_frac_test': 0.5}
#{'model_type': 'json-nn', 'dataset': 'poker', 'lr': 0.001, #97.76
#'mem_dim': 32, 'batch_size': 4, 'epochs': 50, 'poker_frac_test': 1.0},#0.9}
#{'model_type': 'set-nn-max', 'dataset': 'poker', 'lr': 0.001, #83.4
#'mem_dim': 32, 'batch_size': 4, 'epochs': 100, 'poker_frac_test': 0.05},
#{'model_type': 'set-nn-max', 'dataset': 'poker', 'lr': 0.001, #92.24
#'mem_dim': 128, 'batch_size': 64, 'epochs': 100, 'poker_frac_test': 0.2},
#{'model_type': 'set-nn-max', 'dataset': 'poker', 'lr': 0.00025, #97.05
#'mem_dim': 128, 'batch_size': 4, 'epochs': 50, 'poker_frac_test': 0.5},
#{'model_type': 'set-nn-max', 'dataset': 'poker', 'lr': 0.00025, #97.12
#'mem_dim': 64, 'batch_size': 4, 'epochs': 50, 'poker_frac_test': 1.0},#0.9}
#{'model_type': 'mlp', 'dataset': 'poker', 'lr': 0.001, 'mem_dim': 128, #47.8
#'batch_size': 4, 'epochs': 100, 'layers': 5, 'poker_frac_test': 0.05},
#{'model_type': 'mlp', 'dataset': 'poker', 'lr': 0.00025, 'mem_dim': 128, #84.3
#'batch_size': 16, 'epochs': 100, 'layers': 3, 'poker_frac_test': 0.2},
#{'model_type': 'mlp', 'dataset': 'poker', 'lr': 0.00025, 'mem_dim': 32, #97.04
#'batch_size': 4, 'epochs': 50, 'layers': 5, 'poker_frac_test': 0.5},
#{'model_type': 'mlp', 'dataset': 'poker', 'lr': 0.001, 'mem_dim': 32, #98.44
#'batch_size': 4, 'epochs': 50, 'layers': 3, 'poker_frac_test': 1.0}, #0.9}
#{'model_type': 'json-nn-modified', 'dataset': 'poker', 'lr': 0.001, #96.69
#'mem_dim': 64, 'batch_size': 4, 'epochs': 50, 'poker_frac_test': 0.05},
#{'model_type': 'json-nn-modified', 'dataset': 'poker', 'lr': 0.001, #97.30
#'mem_dim': 128, 'batch_size': 16, 'epochs': 50, 'poker_frac_test': 0.2},
#{'model_type': 'json-nn-modified', 'dataset': 'poker', 'lr': 0.001, #97.44
#'mem_dim': 64, 'batch_size': 4, 'epochs': 50, 'poker_frac_test': 0.5},
#{'model_type': 'json-nn-modified', 'dataset': 'poker', 'lr': 0.001, #97.32
#'mem_dim': 128, 'batch_size': 64, 'epochs': 30, 'poker_frac_test': 1.0}, #0.9}
]
seed = "12345678901234567890"
for params in hparams:
for i in range(5):
starttime = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
args = params.copy()
args['test'] = True
args['seed'] = int(seed[:i+3])
print("args:{}\n".format(args))
if args['dataset'] == 'poker':
loss, acc = run_once_poker(**args)
else:
loss, acc = run_once(**args)
outfile = "results_other_{}_{}.txt".format(args['dataset'], args['model_type'])
endtime = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
with open(outfile, 'a+') as f:
f.write("{} -> {}\n".format(starttime, endtime))
f.write("args: {}\n".format(args))
f.write("test_loss: {}\n".format(loss))
f.write("test_acc: {}\n\n\n".format(acc))