/
evaluation_d.py
115 lines (102 loc) · 3.98 KB
/
evaluation_d.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
import matplotlib.pyplot as plt
import numpy as np
from fairml import beta, compute_chain, eta, interval_chaining, top_interval
def main():
c_vals = [1.0, 2.0, 5.0, 10.0]
# Plot: Varying d (confidence)
d_vals = range(1, 50)
k = 2
T = 1000
results = {
'0': {
'ylabel': 'Average regret - TI',
'name': 'avg_regret_ti'
},
'1': {
'ylabel': 'Average regret - IC',
'name': 'avg_regret_ic'
},
'2': {
'ylabel': 'Average regret difference (TI - IC)',
'name': 'avg_regret_diff'
},
'3': {
'ylabel': 'Cumulative regret - TI',
'name': 'cum_regret_ti'
},
'4': {
'ylabel': 'Cumulative regret - IC',
'name': 'cum_regret_ic'
},
'5': {
'ylabel': 'Cumulative regret difference (TI - IC)',
'name': 'cum_regret_diff'
},
'6': {
'ylabel': 'Final regret - TI',
'name': 'final_regret_ti'
},
'7': {
'ylabel': 'Final regret - IC',
'name': 'final_regret_ic'
},
'8': {
'ylabel': 'Final regret difference (TI - IC)',
'name': 'final_regret_diff'
}
}
for _, v in results.items(): # 9 sets of results.
for j in c_vals:
v[str(j)] = []
for c in c_vals:
for d in d_vals:
cum_regret_tis = []
avg_regret_tis = []
final_regret_tis = []
cum_regret_ics = []
avg_regret_ics = []
final_regret_ics = []
for i in range(0, 50): # 500 trials.
X = np.random.uniform(0, 1, size=(T, k, d))
B = beta(k, d, c)
Y = np.array([np.diag(X[t].dot(np.transpose(B))) for t in range(T)])
cum_regret_ti, avg_regret_ti, final_regret_ti = top_interval(
X, Y, k, d, 0.05, T, _print_progress=False)
cum_regret_ic, avg_regret_ic, final_regret_ic = interval_chaining(
X, Y, c, k, d, 0.05, T, _print_progress=False)
cum_regret_tis.append(cum_regret_ti)
avg_regret_tis.append(avg_regret_ti)
final_regret_tis.append(final_regret_ti)
cum_regret_ics.append(cum_regret_ic)
avg_regret_ics.append(avg_regret_ic)
final_regret_ics.append(final_regret_ic)
cum_regret_ti = mean(cum_regret_tis)
avg_regret_ti = mean(avg_regret_tis)
final_regret_ti = mean(avg_regret_tis)
cum_regret_ic = mean(cum_regret_ics)
avg_regret_ic = mean(avg_regret_ics)
final_regret_ics = mean(final_regret_ics)
results['0'][str(c)].append(avg_regret_ti)
results['1'][str(c)].append(avg_regret_ic)
results['2'][str(c)].append(abs(avg_regret_ti - avg_regret_ic))
results['3'][str(c)].append(cum_regret_ti)
results['4'][str(c)].append(cum_regret_ic)
results['5'][str(c)].append(abs(cum_regret_ti - cum_regret_ic))
results['6'][str(c)].append(final_regret_ti)
results['7'][str(c)].append(final_regret_ic)
results['8'][str(c)].append(abs(final_regret_ti - final_regret_ic))
for k, v in results.items():
plt.clf()
c1, = plt.plot(d_vals, results[k]['1.0'], label='c=1')
c2, = plt.plot(d_vals, results[k]['2.0'], label='c=2')
c5, = plt.plot(d_vals, results[k]['5.0'], label='c=5')
c10, = plt.plot(d_vals, results[k]['10.0'], label='c=10')
plt.xticks(np.arange(min(d_vals), max(d_vals) + 1, 10))
plt.legend(handles=[c1, c2, c5, c10])
plt.xlabel('d (# of features)', fontsize=18)
plt.ylabel(v['ylabel'], fontsize=15)
plt.savefig('figures_d_50x/d_50x_' + v['name'])
def mean(numbers):
return float(sum(numbers)) / max(len(numbers), 1)
if __name__ == '__main__':
main()