-
Notifications
You must be signed in to change notification settings - Fork 2
/
scaling_experiment_loss_vs_gen_errosr_normalization.m
144 lines (138 loc) · 5.91 KB
/
scaling_experiment_loss_vs_gen_errosr_normalization.m
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
%clear;clc;
%load('./pytorch_experiments/test_runs_flatness4/loss_vs_gen_errors_norm_frobenius')
%load('./pytorch_experiments/test_runs_flatness5_ProperOriginalExpt/loss_vs_gen_errors_norm_frobenius_final')
%load('./pytorch_experiments/test_runs_flatness5_ProperOriginalExpt/loss_vs_gen_errors_norm_l1')
%load('./pytorch_experiments/test_runs_flatness5_ProperOriginalExpt/loss_vs_gen_errors_norm_l1_divided_by_10')
%load('./pytorch_experiments/test_runs_flatness5_ProperOriginalExpt/loss_vs_gen_errors_norm_l1_divided_by_100')RL_corruption_1.0_loss_vs_gen_errors_norm_l2
%load('./pytorch_experiments/test_runs_flatness5_ProperOriginalExpt/RL_corruption_1.0_loss_vs_gen_errors_norm_l2')
%%
markers = corruption_all_probs;
%markers = std_inits_all;
%%%%RLs = 8:19;
RLs = 62:73;
train_all_losses_normalized(RLs) = train_all_losses_normalized_rand(RLs);
test_all_losses_normalized(RLs) = test_all_losses_normalized_rand(RLs);
train_all_errors_unnormalized(RLs) = train_all_errors_unnormalized_rand(RLs);
gen_all_errors_unnormalized(RLs) = gen_all_errors_unnormalized_rand(RLs);
train_all_losses_unnormalized(RLs) = train_all_losses_unnormalized_rand(RLs);
test_all_losses_unnormalized(RLs) = test_all_losses_unnormalized_rand(RLs);
%% test error vs train error
fig0 = figure;
%lscatter(train_all_errors_unnormalized,gen_all_errors_unnormalized,markers)
lscatter(train_all_errors_unnormalized,gen_all_errors_unnormalized,markers)
xlim([-0.05,1])
xlabel('Train Error (Network Normalized)')
ylabel('Test Error (Network Normalized)')
%lsline
xlim([-0.05,1])
xlabel('Train Error (Network Normalized)')
ylabel('Test Error (Network Normalized)')
%% test error vs train loss (all normalized)
fig1 = figure;
lscatter(train_all_losses_normalized,gen_all_errors_normalized,markers)
%lscatter(all_train_errors,gen_all_errors_normalized,markers)
%lsline
%title('The weights of all models are normalized')
xlabel('Train Loss (Network Normalized)')
ylabel('Test Error (Network Normalized)')
%% IMPORTANT: test loss vs train loss (all normalized) - shows the linear correlation of the train loss and test loss
% fig2 = figure;
% lscatter(train_all_losses_normalized,test_all_losses_normalized,markers)
% %title('The weights of all models are normalized')
% xlabel('Train Loss (Network Normalized)')
% ylabel('Test Loss (Network Normalized)')
% hl = lsline;
% B = [ones(size(hl.XData(:))), hl.XData(:)]\hl.YData(:);
% Slope = B(2)
% Intercept = B(1)
% X = train_all_losses_normalized;y = test_all_losses_normalized;
% mdl = fitlm(X,y);
% RMSE = mdl.RMSE
% Rsquared_Ordinary = mdl.Rsquared.Ordinary
% Rsquared_adjusted = mdl.Rsquared
%
fig2 = figure;
% X_nl = train_all_losses_normalized(1:7);
% Y_nl = test_all_losses_normalized(1:7);
% X_rl = train_all_losses_normalized(RLs);
% Y_rl = test_all_losses_normalized(RLs);
% X = [X_nl X_rl];
% Y = [Y_nl Y_rl];
% markers_XY = [markers(1:7) markers(RLs)];
% lscatter(X,Y,markers_XY)
%c = repelem([0 0.4470 0.7410],length(X),1);
c = [0 0.4470 0.7410];
%scatter(X,Y,c)
%scatter(X,Y,[0 0.4470 0.7410])
%scatter(X,Y)
%scatter(X,Y,'MarkerEdgeColor',c)
%hold;
%scatter(train_all_losses_normalized,test_all_losses_normalized)
lscatter(train_all_losses_normalized,test_all_losses_normalized,markers)
hold;
hl = lsline;
%title('The weights of all models are normalized')
B = [ones(size(hl.XData(:))), hl.XData(:)]\hl.YData(:);
Slope = B(2)
Intercept = B(1)
X = train_all_losses_normalized;y = test_all_losses_normalized;
mdl = fitlm(X,y);
RMSE = mdl.RMSE
Rsquared_Ordinary = mdl.Rsquared.Ordinary
Rsquared_adjusted = mdl.Rsquared
xlabel('Train Loss (Network Normalized)')
ylabel('Test Loss (Network Normalized)')
%% IMPORTANT: test error (unormalized) vs train loss (normalized), this checks if we can predict test error from train loss
fig3 = figure;
lscatter(train_all_losses_normalized,gen_all_errors_unnormalized,markers)
%lsline
%title('Train Loss vs Test Error')
xlabel('Train Loss (Network Normalized)')
ylabel('Test Error (Network Unnormalized)')
%% test loss (unormalized) vs train loss (normalized), not interesting cuz unnormalized loss diverges to infinity
fig4 = figure;
lscatter(train_all_losses_normalized,test_all_losses_unnormalized,markers)
%lsline
%title('Train Loss vs Test Loss')
xlabel('Train Loss (Network Normalized)')
ylabel('Test Loss (Network Unnormalized)')
%% CONTROL1: test loss (unormalized) vs train loss (unnormalized)
fig5 = figure;
lscatter(train_all_losses_unnormalized,test_all_losses_unnormalized,markers)
% h = scatter(train_all_losses_unnormalized,test_all_losses_unnormalized)
% c = get(h,'Color')
% c{1}
% c{2}
%lsline
title('Control 1: The weights of all models are unnormalized')
xlabel('Train Loss (Network Unnormalized)')
ylabel('Test Loss (Network Unnormalized)')
xlim([-2.5e-4,9e-3])
%% CONTROL2: test error (unormalized) vs train loss (unnormalized)
fig6 = figure;
lscatter(train_all_losses_unnormalized,gen_all_errors_unnormalized,markers)
%lsline
title('Control 2: The weights of all models are unnormalized')
xlabel('Train Loss (Network Unnormalized)')
ylabel('Test Error (Network Unnormalized)')
%% save
saveas(fig0,'test_error_vs_train_error_all_unnormalized');
saveas(fig0,'test_error_vs_train_error_all_unnormalized','pdf');
%
saveas(fig1,'test_error_vs_train_loss_all_normalized');
saveas(fig1,'test_error_vs_train_loss_all_normalized','pdf');
%
saveas(fig2,'important_test_loss_vs_train_loss_all_normalized');
saveas(fig2,'important_test_loss_vs_train_loss_all_normalized','pdf');
%
saveas(fig3,'important_test_error_vs_train_loss_unnormalized_vs_normalized');
saveas(fig3,'important_test_error_vs_train_loss_unnormalized_vs_normalized','pdf');
%
saveas(fig4,'test_loss_vs_train_loss_unnormalized_vs_normalized');
saveas(fig4,'test_loss_vs_train_loss_unnormalized_vs_normalized','pdf');
%
saveas(fig5,'control1_test_error_vs_train_loss_all_unnormalized');
saveas(fig5,'control1_test_error_vs_train_loss_all_unnormalized','pdf');
%
saveas(fig6,'control2_test_error_vs_train_loss_all_unnormalized');
saveas(fig6,'control2_test_error_vs_train_loss_all_unnormalized','pdf');