Introduce counter for f_tol (IPNewton support) (#533)

* Introduce counter for f_tol (IPNewton support)

* Fix typo

src/multivariate/optimize/interface.jl

@@ -17,9 +17,6 @@ function check_kwargs(kwargs, fallback_method)
 end
 
 default_options(method::AbstractOptimizer) = Dict{Symbol, Any}()
-function default_options(method::AbstractNGMRES)
-    Dict(:allow_f_increases => true)
-end
 
 function add_default_opts!(opts::Dict{Symbol, Any}, method::AbstractOptimizer)
     for newopt in default_options(method)

src/multivariate/optimize/optimize.jl

@@ -19,9 +19,9 @@ function initial_convergence(d, state, method::AbstractOptimizer, initial_x, opt
 end
 initial_convergence(d, state, method::ZerothOrderOptimizer, initial_x, options) = false
 
-function optimize(d::D, initial_x::AbstractArray{Tx, N}, method::M,
+function optimize(d::D, initial_x::Tx, method::M,
                   options::Options = Options(;default_options(method)...),
-                  state = initial_state(method, options, d, complex_to_real(d, initial_x))) where {D<:AbstractObjective, M<:AbstractOptimizer, Tx, N}
+                  state = initial_state(method, options, d, complex_to_real(d, initial_x))) where {D<:AbstractObjective, M<:AbstractOptimizer, Tx <: AbstractArray}
     if length(initial_x) == 1 && typeof(method) <: NelderMead
         error("You cannot use NelderMead for univariate problems. Alternatively, use either interval bound univariate optimization, or another method such as BFGS or Newton.")
     end
@@ -30,12 +30,11 @@ function optimize(d::D, initial_x::AbstractArray{Tx, N}, method::M,
 
     initial_x = complex_to_real(d, initial_x)
 
-    n = length(initial_x)
     tr = OptimizationTrace{typeof(value(d)), typeof(method)}()
     tracing = options.store_trace || options.show_trace || options.extended_trace || options.callback != nothing
     stopped, stopped_by_callback, stopped_by_time_limit = false, false, false
     f_limit_reached, g_limit_reached, h_limit_reached = false, false, false
-    x_converged, f_converged, f_increased = false, false, false
+    x_converged, f_converged, f_increased, counter_f_tol = false, false, false, 0
 
     g_converged = initial_convergence(d, state, method, initial_x, options)
     converged = g_converged
@@ -50,9 +49,13 @@ function optimize(d::D, initial_x::AbstractArray{Tx, N}, method::M,
         iteration += 1
 
         update_state!(d, state, method) && break # it returns true if it's forced by something in update! to stop (eg dx_dg == 0.0 in BFGS, or linesearch errors)
-        update_g!(d, state, method)
+        update_g!(d, state, method) # TODO: Should this be `update_fg!`?
         x_converged, f_converged,
         g_converged, converged, f_increased = assess_convergence(state, d, options)
+        # For some problems it may be useful to require `f_converged` to be hit multiple times
+        # TODO: Do the same for x_tol?
+        counter_f_tol = f_converged ? counter_f_tol+1 : 0
+        converged = converged | (counter_f_tol > options.successive_f_tol)
 
         !converged && update_h!(d, state, method) # only relevant if not converged
 
@@ -61,6 +64,8 @@ function optimize(d::D, initial_x::AbstractArray{Tx, N}, method::M,
             stopped_by_callback = trace!(tr, d, state, iteration, method, options)
         end
 
+        # Check time_limit; if none is provided it is NaN and the comparison
+        # will always return false.
         stopped_by_time_limit = time()-t0 > options.time_limit ? true : false
         f_limit_reached = options.f_calls_limit > 0 && f_calls(d) >= options.f_calls_limit ? true : false
         g_limit_reached = options.g_calls_limit > 0 && g_calls(d) >= options.g_calls_limit ? true : false

src/multivariate/solvers/first_order/ngmres.jl

@@ -431,3 +431,7 @@ end
 function assess_convergence(state::NGMRESState, d, options)
     default_convergence_assessment(state, d, options)
 end
+
+function default_options(method::AbstractNGMRES)
+    Dict(:allow_f_increases => true)
+end

src/types.jl

@@ -15,6 +15,7 @@ struct Options{T, TCallback}
     g_calls_limit::Int
     h_calls_limit::Int
     allow_f_increases::Bool
+    successive_f_tol::Int
     iterations::Int
     store_trace::Bool
     show_trace::Bool
@@ -32,6 +33,7 @@ function Options(;
         g_calls_limit::Int = 0,
         h_calls_limit::Int = 0,
         allow_f_increases::Bool = false,
+        successive_f_tol::Int = 0,
         iterations::Integer = 1_000,
         store_trace::Bool = false,
         show_trace::Bool = false,
@@ -44,7 +46,7 @@ function Options(;
     #    show_trace = true
     #end
     Options(promote(x_tol, f_tol, g_tol)..., f_calls_limit, g_calls_limit, h_calls_limit,
-        allow_f_increases, Int(iterations), store_trace, show_trace, extended_trace,
+        allow_f_increases, successive_f_tol, Int(iterations), store_trace, show_trace, extended_trace,
         Int(show_every), callback, time_limit)
 end
 

src/utilities/assess_convergence.jl

@@ -5,71 +5,69 @@ x_abschange(x, x_previous) = maxdiff(x, x_previous)
 g_residual(d::AbstractObjective) = g_residual(gradient(d))
 g_residual(d::NonDifferentiable) = convert(typeof(value(d)), NaN)
 g_residual(g) = vecnorm(g, Inf)
+gradient_convergence_assessment(state::AbstractOptimizerState, d, options) = g_residual(gradient(d)) ≤ options.g_tol
+gradient_convergence_assessment(state::ZerothOrderState, d, options) = false
 
-# Used by fminbox
-function assess_convergence(x,
-                            x_previous,
-                            f_x,
-                            f_x_previous,
-                            g,
-                            x_tol,
-                            f_tol,
-                            g_tol)
+# Default function for convergence assessment used by
+# AcceleratedGradientDescentState, BFGSState, ConjugateGradientState,
+# GradientDescentState, LBFGSState, MomentumGradientDescentState and NewtonState
+function default_convergence_assessment(state::AbstractOptimizerState, d, options)
     x_converged, f_converged, f_increased, g_converged = false, false, false, false
 
-    if x_abschange(x, x_previous) ≤ x_tol
+    # TODO: Create function for x_convergence_assessment
+    if x_abschange(state.x, state.x_previous) ≤ options.x_tol
         x_converged = true
     end
 
-    # Absolute Tolerance
-    # if abs(f_x - f_x_previous) < f_tol
     # Relative Tolerance
-    if f_abschange(f_x, f_x_previous) ≤ f_tol*abs(f_x)
+    # TODO: Create function for f_convergence_assessment
+    f_x = value(d)
+    if f_abschange(f_x, state.f_x_previous) ≤ options.f_tol*abs(f_x)
         f_converged = true
     end
 
-    if f_x > f_x_previous
+    if f_x > state.f_x_previous
         f_increased = true
     end
 
-    if g_residual(g) ≤ g_tol
-        g_converged = true
-    end
+    g_converged = gradient_convergence_assessment(state,d,options)
 
     converged = x_converged || f_converged || g_converged
 
     return x_converged, f_converged, g_converged, converged, f_increased
 end
 
-# Default function for convergence assessment used by
-# AcceleratedGradientDescentState, BFGSState, ConjugateGradientState,
-# GradientDescentState, LBFGSState, MomentumGradientDescentState and NewtonState
-function default_convergence_assessment(state::AbstractOptimizerState, d, options)
+# Used by Fminbox and IPNewton
+function assess_convergence(x,
+                            x_previous,
+                            f_x,
+                            f_x_previous,
+                            g,
+                            x_tol,
+                            f_tol,
+                            g_tol)
     x_converged, f_converged, f_increased, g_converged = false, false, false, false
 
-    if x_abschange(state.x, state.x_previous) ≤ options.x_tol
+    if x_abschange(x, x_previous) ≤ x_tol
         x_converged = true
     end
 
     # Absolute Tolerance
     # if abs(f_x - f_x_previous) < f_tol
     # Relative Tolerance
-    f_x = value(d)
-    if f_abschange(f_x, state.f_x_previous) ≤ options.f_tol*abs(f_x)
+    if f_abschange(f_x, f_x_previous) ≤ f_tol*abs(f_x)
         f_converged = true
     end
 
-    if f_x > state.f_x_previous
+    if f_x > f_x_previous
         f_increased = true
     end
 
-    g_converged = gradient_convergence_assessment(state,d,options)
-
+    if g_residual(g) ≤ g_tol
+        g_converged = true
+    end
+
     converged = x_converged || f_converged || g_converged
 
     return x_converged, f_converged, g_converged, converged, f_increased
 end
-
-gradient_convergence_assessment(state::AbstractOptimizerState, d, options) = g_residual(gradient(d)) ≤ options.g_tol
-gradient_convergence_assessment(state::ZerothOrderState, d, options) = false
-