Skip to content

Commit a8a87dd

Browse files
kthohrkthohr
committed
update OptimLib
1 parent b8ae1cb commit a8a87dd

File tree

10 files changed

+258
-88
lines changed

10 files changed

+258
-88
lines changed

‎include/optim/misc/optim_options.hpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
#endif
2828

2929
#ifndef OPTIM_VERSION_MINOR
30-
#define OPTIM_VERSION_MINOR 1
30+
#define OPTIM_VERSION_MINOR 2
3131
#endif
3232

3333
#ifndef OPTIM_VERSION_PATCH
@@ -78,6 +78,7 @@
7878

7979
namespace optim
8080
{
81+
static const double eps_dbl = std::numeric_limits<double>::epsilon();
8182
static const double inf = std::numeric_limits<double>::infinity();
8283
using uint_t = unsigned int;
8384
}

‎include/optim/misc/optim_structs.hpp

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,26 @@ struct gd_settings_t
2929
{
3030
// step size, or 'learning rate'
3131
double step_size = 0.1;
32+
33+
// decay
3234
bool step_decay = false;
35+
3336
uint_t step_decay_periods = 10;
3437
double step_decay_val = 0.5;
3538

3639
// momentum parameter
3740
double momentum_par = 0.9;
41+
42+
// Ada parameters
43+
double norm_term = 10e-08;
44+
45+
double ada_rho = 0.9;
46+
47+
bool ada_max = false;
48+
49+
// Adam parameters
50+
double adam_beta_1 = 0.9;
51+
double adam_beta_2 = 0.999;
3852
};
3953

4054
struct algo_settings_t
@@ -88,7 +102,7 @@ struct algo_settings_t
88102
arma::vec de_initial_ub; // this will default to 0.5
89103

90104
// GD
91-
int gd_method = 1;
105+
int gd_method = 0;
92106
gd_settings_t gd_settings;
93107

94108
// L-BFGS

‎include/optim/misc/transform_vals.hpp

Lines changed: 44 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
## limitations under the License.
1818
##
1919
################################################################################*/
20-
20+
2121
/*
2222
* transform values
2323
*/
@@ -38,13 +38,13 @@ transform(const arma::vec& vals_inp, const arma::uvec& bounds_type, const arma::
3838
vals_trans_out(i) = vals_inp(i);
3939
break;
4040
case 2: // lower bound only
41-
vals_trans_out(i) = std::log(vals_inp(i) - lower_bounds(i));
41+
vals_trans_out(i) = std::log(vals_inp(i) - lower_bounds(i) + eps_dbl);
4242
break;
4343
case 3: // upper bound only
44-
vals_trans_out(i) = - std::log(upper_bounds(i) - vals_inp(i));
44+
vals_trans_out(i) = - std::log(upper_bounds(i) - vals_inp(i) + eps_dbl);
4545
break;
4646
case 4: // upper and lower bounds
47-
vals_trans_out(i) = std::log(vals_inp(i) - lower_bounds(i)) - std::log(upper_bounds(i) - vals_inp(i));
47+
vals_trans_out(i) = std::log(vals_inp(i) - lower_bounds(i) + eps_dbl) - std::log(upper_bounds(i) - vals_inp(i) + eps_dbl);
4848
break;
4949
}
5050
}
@@ -70,26 +70,56 @@ inv_transform(const arma::vec& vals_trans_inp, const arma::uvec& bounds_type, co
7070
vals_out(i) = vals_trans_inp(i);
7171
break;
7272
case 2: // lower bound only
73-
vals_out(i) = lower_bounds(i) + std::exp(vals_trans_inp(i));
73+
if (!std::isfinite(vals_trans_inp(i)))
74+
{
75+
vals_out(i) = lower_bounds(i) + eps_dbl;
76+
}
77+
else
78+
{
79+
vals_out(i) = lower_bounds(i) + eps_dbl + std::exp(vals_trans_inp(i));
80+
}
7481
break;
7582
case 3: // upper bound only
76-
vals_out(i) = upper_bounds(i) - std::exp(-vals_trans_inp(i));
83+
if (!std::isfinite(vals_trans_inp(i)))
84+
{
85+
vals_out(i) = upper_bounds(i) - eps_dbl;
86+
}
87+
else
88+
{
89+
vals_out(i) = upper_bounds(i) - eps_dbl - std::exp(-vals_trans_inp(i));
90+
}
7791
break;
7892
case 4: // upper and lower bounds
79-
if (!std::isfinite(vals_trans_inp(i))) {
80-
if (vals_trans_inp(i) < 0.0) {
81-
vals_out(i) = lower_bounds(i);
82-
} else {
83-
vals_out(i) = upper_bounds(i);
93+
if (!std::isfinite(vals_trans_inp(i)))
94+
{
95+
if (std::isnan(vals_trans_inp(i)))
96+
{
97+
vals_out(i) = (upper_bounds(i) - lower_bounds(i)) / 2.0;
98+
}
99+
else if (vals_trans_inp(i) < 0.0)
100+
{
101+
vals_out(i) = lower_bounds(i) + eps_dbl;
102+
}
103+
else
104+
{
105+
vals_out(i) = upper_bounds(i) - eps_dbl;
106+
}
107+
}
108+
else
109+
{
110+
vals_out(i) = ( lower_bounds(i) + eps_dbl + (upper_bounds(i) - eps_dbl)*std::exp(vals_trans_inp(i)) ) \
111+
/ ( 1.0 + std::exp(vals_trans_inp(i)) );
112+
113+
if (!std::isfinite(vals_out(i)))
114+
{
115+
vals_out(i) = upper_bounds(i) - eps_dbl;
84116
}
85-
} else {
86-
vals_out(i) = ( lower_bounds(i) + upper_bounds(i)*std::exp(vals_trans_inp(i)) ) / ( 1 + std::exp(vals_trans_inp(i)) );
87117
}
88118
break;
89119
}
90120
}
91121

92122
//
93-
123+
94124
return vals_out;
95125
}

‎include/optim/unconstrained/gd.hpp

Lines changed: 135 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,140 @@ bool gd(arma::vec& init_out_vals, std::function<double (const arma::vec& vals_in
3131
bool gd(arma::vec& init_out_vals, std::function<double (const arma::vec& vals_inp, arma::vec* grad_out, void* opt_data)> opt_objfn, void* opt_data, algo_settings_t& settings);
3232

3333
// internal update function
34-
arma::vec gd_update(const arma::vec& grad, const arma::vec& grad_p, const arma::vec& direc,
35-
const uint_t iter, const uint_t gd_method_inp, gd_settings_t& gd_settings);
34+
35+
inline
36+
arma::vec
37+
gd_update(const arma::vec& vals_inp, const arma::vec& grad, const arma::vec& grad_p, const arma::vec& direc,
38+
std::function<double (const arma::vec& vals_inp, arma::vec* grad_out, void* opt_data)> box_objfn, void* opt_data,
39+
const uint_t iter, const uint_t gd_method, gd_settings_t& gd_settings,
40+
arma::vec& adam_vec_m, arma::vec& adam_vec_v)
41+
{
42+
arma::vec direc_out; // direction
43+
44+
if (gd_settings.step_decay)
45+
{
46+
if (iter % gd_settings.step_decay_periods == 0)
47+
{
48+
gd_settings.step_size *= gd_settings.step_decay_val;
49+
}
50+
}
51+
52+
switch (gd_method)
53+
{
54+
case 0: // basic
55+
{
56+
direc_out = gd_settings.step_size * grad_p;
57+
break;
58+
}
59+
60+
case 1: // momentum
61+
{
62+
// direc_out = gd_settings.step_size * (gd_settings.momentum_par * direc + grad_p);
63+
direc_out = gd_settings.momentum_par * direc + gd_settings.step_size * grad_p;
64+
break;
65+
}
66+
67+
case 2: // Nesterov accelerated gradient
68+
{
69+
arma::vec NAG_grad(vals_inp.n_elem);
70+
box_objfn(vals_inp - gd_settings.momentum_par * direc, &NAG_grad, opt_data);
71+
72+
// direc_out = gd_settings.step_size * (gd_settings.momentum_par * direc + NAG_grad);
73+
direc_out = gd_settings.momentum_par * direc + gd_settings.step_size * NAG_grad;
74+
break;
75+
}
76+
77+
case 3: // AdaGrad
78+
{
79+
adam_vec_v += arma::pow(grad_p,2);
80+
81+
direc_out = gd_settings.step_size * grad_p / (arma::sqrt(adam_vec_v) + gd_settings.norm_term);
82+
break;
83+
}
84+
85+
case 4: // RMSProp
86+
{
87+
adam_vec_v = gd_settings.ada_rho * adam_vec_v + (1.0 - gd_settings.ada_rho) * arma::pow(grad_p,2);
88+
89+
direc_out = gd_settings.step_size * grad_p / (arma::sqrt(adam_vec_v) + gd_settings.norm_term);
90+
break;
91+
}
92+
93+
case 5: // Adadelta
94+
{
95+
if (iter == 1) {
96+
adam_vec_m += gd_settings.step_size;
97+
}
98+
adam_vec_v = gd_settings.ada_rho * adam_vec_v + (1.0 - gd_settings.ada_rho) * arma::pow(grad_p,2);
99+
100+
direc_out = grad_p % (arma::sqrt(adam_vec_m) + gd_settings.norm_term) / (arma::sqrt(adam_vec_v) + gd_settings.norm_term);
101+
102+
adam_vec_m = gd_settings.ada_rho * adam_vec_m + (1.0 - gd_settings.ada_rho) * arma::pow(direc_out,2);
103+
break;
104+
}
105+
106+
case 6: // Adam and AdaMax
107+
{
108+
adam_vec_m = gd_settings.adam_beta_1 * adam_vec_m + (1.0 - gd_settings.adam_beta_1) * grad_p;
109+
110+
if (gd_settings.ada_max)
111+
{
112+
adam_vec_v = arma::max(gd_settings.adam_beta_2 * adam_vec_v, arma::abs(grad_p));
113+
114+
double adam_step_size = gd_settings.step_size / (1.0 - std::pow(gd_settings.adam_beta_1,iter));
115+
116+
direc_out = adam_step_size * adam_vec_m / (adam_vec_v + gd_settings.norm_term);
117+
}
118+
else
119+
{
120+
double adam_step_size = gd_settings.step_size * std::sqrt(1.0 - std::pow(gd_settings.adam_beta_2,iter)) \
121+
/ (1.0 - std::pow(gd_settings.adam_beta_1,iter));
122+
123+
adam_vec_v = gd_settings.adam_beta_2 * adam_vec_v + (1.0 - gd_settings.adam_beta_2) * arma::pow(grad_p,2);
124+
125+
direc_out = adam_step_size * adam_vec_m / (arma::sqrt(adam_vec_v) + gd_settings.norm_term);
126+
}
127+
128+
break;
129+
}
130+
131+
case 7: // Nadam and NadaMax
132+
{
133+
adam_vec_m = gd_settings.adam_beta_1 * adam_vec_m + (1.0 - gd_settings.adam_beta_1) * grad_p;
134+
135+
if (gd_settings.ada_max)
136+
{
137+
adam_vec_v = arma::max(gd_settings.adam_beta_2 * adam_vec_v, arma::abs(grad_p));
138+
139+
arma::vec m_hat = adam_vec_m / (1.0 - std::pow(gd_settings.adam_beta_1,iter));
140+
arma::vec grad_hat = grad_p / (1.0 - std::pow(gd_settings.adam_beta_1,iter));
141+
142+
direc_out = gd_settings.step_size * ( gd_settings.adam_beta_1 * m_hat + (1.0 - gd_settings.adam_beta_1) * grad_hat ) \
143+
/ (adam_vec_v + gd_settings.norm_term);
144+
}
145+
else
146+
{
147+
adam_vec_v = gd_settings.adam_beta_2 * adam_vec_v + (1.0 - gd_settings.adam_beta_2) * arma::pow(grad_p,2);
148+
149+
arma::vec m_hat = adam_vec_m / (1.0 - std::pow(gd_settings.adam_beta_1,iter));
150+
arma::vec v_hat = adam_vec_v / (1.0 - std::pow(gd_settings.adam_beta_2,iter));
151+
arma::vec grad_hat = grad_p / (1.0 - std::pow(gd_settings.adam_beta_1,iter));
152+
153+
direc_out = gd_settings.step_size * ( gd_settings.adam_beta_1 * m_hat + (1.0 - gd_settings.adam_beta_1) * grad_hat ) \
154+
/ (arma::sqrt(v_hat) + gd_settings.norm_term);
155+
}
156+
157+
break;
158+
}
159+
160+
default:
161+
{
162+
printf("error: unknown value for gd_method");
163+
break;
164+
}
165+
}
166+
167+
return direc_out;
168+
}
36169

37170
#endif

‎src/optim/unconstrained/bfgs.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,11 @@ optim::bfgs_int(arma::vec& init_out_vals, std::function<double (const arma::vec&
7373

7474
ret = opt_objfn(vals_inv_trans,&grad_obj,opt_data);
7575

76-
arma::mat jacob_matrix = jacobian_adjust(vals_inp,bounds_type,lower_bounds,upper_bounds);
76+
// arma::mat jacob_matrix = jacobian_adjust(vals_inp,bounds_type,lower_bounds,upper_bounds);
77+
arma::vec jacob_vec = arma::diagvec(jacobian_adjust(vals_inp,bounds_type,lower_bounds,upper_bounds));
7778

78-
*grad_out = jacob_matrix * grad_obj; // no need for transpose as jacob_matrix is diagonal
79+
// *grad_out = jacob_matrix * grad_obj; // no need for transpose as jacob_matrix is diagonal
80+
*grad_out = jacob_vec % grad_obj;
7981
}
8082
else
8183
{

‎src/optim/unconstrained/cg.cpp

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -65,30 +65,33 @@ optim::cg_int(arma::vec& init_out_vals, std::function<double (const arma::vec& v
6565
= [opt_objfn, vals_bound, bounds_type, lower_bounds, upper_bounds] (const arma::vec& vals_inp, arma::vec* grad_out, void* opt_data) \
6666
-> double
6767
{
68-
if (vals_bound) {
69-
68+
if (vals_bound)
69+
{
7070
arma::vec vals_inv_trans = inv_transform(vals_inp, bounds_type, lower_bounds, upper_bounds);
71-
7271
double ret;
7372

74-
if (grad_out) {
73+
if (grad_out)
74+
{
7575
arma::vec grad_obj = *grad_out;
7676

7777
ret = opt_objfn(vals_inv_trans,&grad_obj,opt_data);
7878

79-
arma::mat jacob_matrix = jacobian_adjust(vals_inp,bounds_type,lower_bounds,upper_bounds);
79+
// arma::mat jacob_matrix = jacobian_adjust(vals_inp,bounds_type,lower_bounds,upper_bounds);
80+
arma::vec jacob_vec = arma::diagvec(jacobian_adjust(vals_inp,bounds_type,lower_bounds,upper_bounds));
8081

81-
// *grad_out = jacob_matrix.t() * grad_obj; // correct gradient for transformation
82-
*grad_out = jacob_matrix * grad_obj; // no need for transpose as jacob_matrix is diagonal
83-
} else {
82+
// *grad_out = jacob_matrix * grad_obj; // no need for transpose as jacob_matrix is diagonal
83+
*grad_out = jacob_vec % grad_obj;
84+
}
85+
else
86+
{
8487
ret = opt_objfn(vals_inv_trans,nullptr,opt_data);
8588
}
8689

8790
return ret;
88-
} else {
89-
double ret = opt_objfn(vals_inp,grad_out,opt_data);
90-
91-
return ret;
91+
}
92+
else
93+
{
94+
return opt_objfn(vals_inp,grad_out,opt_data);
9295
}
9396
};
9497

‎src/optim/unconstrained/de_prmm.cpp

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,11 +82,14 @@ optim::de_prmm_int(arma::vec& init_out_vals, std::function<double (const arma::v
8282
= [opt_objfn, vals_bound, bounds_type, lower_bounds, upper_bounds] (const arma::vec& vals_inp, arma::vec* grad_out, void* opt_data) \
8383
-> double
8484
{
85-
if (vals_bound) {
85+
if (vals_bound)
86+
{
8687
arma::vec vals_inv_trans = inv_transform(vals_inp, bounds_type, lower_bounds, upper_bounds);
8788

8889
return opt_objfn(vals_inv_trans,nullptr,opt_data);
89-
} else {
90+
}
91+
else
92+
{
9093
return opt_objfn(vals_inp,nullptr,opt_data);
9194
}
9295
};

0 commit comments

Comments
 (0)