Fix input for (L)BFGS and PSO (#431)

* Fix input for (L)BFGS ans PSO.

src/multivariate/solvers/first_order/bfgs.jl

@@ -57,7 +57,7 @@ function update_state!{T}(d, state::BFGSState{T}, method::BFGS)
 
     # Set the search direction
     # Search direction is the negative gradient divided by the approximate Hessian
-    A_mul_B!(state.s, state.invH, gradient(d))
+    A_mul_B!(vec(state.s), state.invH, vec(gradient(d)))
     scale!(state.s, -1)
 
     # Maintain a record of the previous gradient
@@ -72,11 +72,9 @@ function update_state!{T}(d, state::BFGSState{T}, method::BFGS)
     copy!(state.x_previous, state.x)
 
     # Update current position
-    @simd for i in 1:n
-        @inbounds state.dx[i] = state.alpha * state.s[i]
-        @inbounds state.x[i] = state.x[i] + state.dx[i]
-    end
-
+    state.dx .= state.alpha.*state.s
+    state.x .= state.x .+ state.dx
+#  
     lssuccess == false # break on linesearch error
 end
 

src/multivariate/solvers/first_order/l_bfgs.jl

@@ -88,11 +88,11 @@ end
 
 Base.summary(::LBFGS) = "L-BFGS"
 
-type LBFGSState{T,N,M}
+type LBFGSState{T,N,M,G}
     x::Array{T,N}
     x_previous::Array{T,N}
-    g_previous::Array{T,N}
-    rho::Array{T,N}
+    g_previous::G
+    rho::Vector{T}
     dx_history::Array{T,M}
     dg_history::Array{T,M}
     dx::Array{T,N}
@@ -112,15 +112,15 @@ function initial_state{T}(method::LBFGS, options, d, initial_x::Array{T})
     LBFGSState(copy(initial_x), # Maintain current state in state.x
               similar(initial_x), # Maintain previous state in state.x_previous
               similar(gradient(d)), # Store previous gradient in state.g_previous
-              Array{T}(method.m), # state.rho
-              Array{T}(n, method.m), # Store changes in position in state.dx_history
-              Array{T}(n, method.m), # Store changes in gradient in state.dg_history
+              Vector{T}(method.m), # state.rho
+              Matrix{T}(n, method.m), # Store changes in position in state.dx_history
+              Matrix{T}(n, method.m), # Store changes in gradient in state.dg_history
               similar(initial_x), # Buffer for new entry in state.dx_history
               similar(initial_x), # Buffer for new entry in state.dg_history
               similar(initial_x), # Buffer stored in state.u
               T(NaN), # Store previous f in state.f_x_previous
               similar(initial_x), #Buffer for use by twoloop
-              Array{T}(method.m), #Buffer for use by twoloop
+              Vector{T}(method.m), #Buffer for use by twoloop
               0,
               similar(initial_x), # Store current search direction in state.s
               @initial_linesearch()...) # Maintain a cache for line search results in state.lsr
@@ -135,9 +135,9 @@ function update_state!{T}(d, state::LBFGSState{T}, method::LBFGS)
     method.precondprep!(method.P, state.x)
 
     # Determine the L-BFGS search direction # FIXME just pass state and method?
-    twoloop!(state.s, gradient(d), state.rho, state.dx_history, state.dg_history,
+    twoloop!(vec(state.s), vec(gradient(d)), vec(state.rho), state.dx_history, state.dg_history,
              method.m, state.pseudo_iteration,
-             state.twoloop_alpha, state.twoloop_q, method.P)
+             state.twoloop_alpha, vec(state.twoloop_q), method.P)
 
     # Maintain a record of previous position
     copy!(state.x_previous, state.x)
@@ -152,10 +152,8 @@ function update_state!{T}(d, state::LBFGSState{T}, method::LBFGS)
     state.f_x_previous = f_x_prev
 
     # Update current position
-    @simd for i in 1:n
-        @inbounds state.dx[i] = state.alpha * state.s[i]
-        @inbounds state.x[i] = state.x[i] + state.dx[i]
-    end
+    state.dx .= state.alpha .* state.s
+    state.x .= state.x .+ state.dx
 
     lssuccess == false # break on linesearch error
 end

src/multivariate/solvers/zeroth_order/particle_swarm.jl

@@ -21,8 +21,8 @@ type ParticleSwarmState{T,N}
     X
     V
     X_best
-    score::Array{T,N}
-    best_score::Array{T,N}
+    score::Vector{T}
+    best_score::Vector{T}
     x_learn
     current_state
     iterations::Int
@@ -46,16 +46,18 @@ function initial_state{T}(method::ParticleSwarm, options, f, initial_x::Array{T}
     the swarm jumping out of local minima.
     =#
     n = length(initial_x)
-    lower = convert(typeof(initial_x), copy(method.lower))
-    upper = convert(typeof(initial_x), copy(method.upper))
-
+    # TODO do we even need a lower((n) upper    ) that is different from method.lower(upper)
     # do some checks on input parameters
-    @assert length(lower) == length(upper) "lower and upper must be of same length."
-    if length(lower) > 0
+    @assert length(method.lower) == length(method.upper) "lower and upper must be of same length."
+    if length(method.lower) > 0
+        lower = copy!(similar(initial_x), copy(method.lower))
+        upper = copy!(similar(initial_x), copy(method.upper))
         limit_search_space = true
         @assert length(lower) == length(initial_x) "limits must be of same length as x_initial."
         @assert all(upper .> lower) "upper must be greater than lower"
     else
+        lower = similar(initial_x)
+        upper = similar(initial_x)
         limit_search_space = false
     end
 
@@ -79,9 +81,9 @@ function initial_state{T}(method::ParticleSwarm, options, f, initial_x::Array{T}
     X_best = Array{T}(n, n_particles)
     dx = zeros(T, n)
     score = zeros(T, n_particles)
-    x = zeros(T, n)
+    x = similar(initial_x)
     best_score = zeros(T, n_particles)
-    x_learn = zeros(T, n)
+    x_learn = similar(initial_x)
 
     current_state = 0
     value!(f, initial_x)

test/multivariate/array.jl

@@ -9,36 +9,34 @@
         return
     end
     @testset "vector" begin
-        for m in (AcceleratedGradientDescent(), ConjugateGradient(), BFGS(), LBFGS(), NelderMead(), GradientDescent(), MomentumGradientDescent(), NelderMead(), SimulatedAnnealing())
-            res = optimize(f, g!, [1., 0., 1., 0.], m)
-            if !(m in (NelderMead(), SimulatedAnnealing()))
-                @test typeof(Optim.minimizer(res)) <: Vector
+        for m in (AcceleratedGradientDescent, ConjugateGradient, BFGS, LBFGS, NelderMead, GradientDescent, MomentumGradientDescent, NelderMead, ParticleSwarm, SimulatedAnnealing)
+            res = optimize(f, g!, [1., 0., 1., 0.], m())
+            @test typeof(Optim.minimizer(res)) <: Vector
+            if !(m in (NelderMead, SimulatedAnnealing))
                 @test vecnorm(Optim.minimizer(res) - [10.0, 0.0, 0.0, 5.0]) < 10e-8
             end
         end
     end
 
-    # PSO does not accept matrix input
     @testset "matrix" begin
-        for m in (AcceleratedGradientDescent(), ConjugateGradient(),  GradientDescent(), MomentumGradientDescent(),  SimulatedAnnealing())
-            res = optimize(f, g!, eye(2), m)
-            if m != SimulatedAnnealing()
-                @test typeof(Optim.minimizer(res)) <: Matrix
+        for m in (AcceleratedGradientDescent, ConjugateGradient, BFGS, LBFGS, ConjugateGradient,  GradientDescent, MomentumGradientDescent, ParticleSwarm, SimulatedAnnealing)
+            res = optimize(f, g!, eye(2), m())
+            @test typeof(Optim.minimizer(res)) <: Matrix
+            if !(m in (SimulatedAnnealing, ParticleSwarm))
                 @test vecnorm(Optim.minimizer(res) - [10.0 0.0; 0.0 5.0]) < 10e-8
             end
         end
     end
 
-    # PSO does not accept tensor input
     @testset "tensor" begin
         eye3 = zeros(2,2,1)
         eye3[:,:,1] = eye(2)
-        for m in (AcceleratedGradientDescent(), ConjugateGradient(),  GradientDescent(), MomentumGradientDescent(),  SimulatedAnnealing())
-            res = optimize(f, g!, eye3, m)
-            if m != SimulatedAnnealing()
-                _minimizer = Optim.minimizer(res)
-                @test typeof(_minimizer) <: Array{Float64, 3}
-                @test size(_minimizer) == (2,2,1)
+        for m in (AcceleratedGradientDescent, ConjugateGradient, BFGS, LBFGS, ConjugateGradient,  GradientDescent, MomentumGradientDescent, ParticleSwarm, SimulatedAnnealing)
+            res = optimize(f, g!, eye3, m())
+            _minimizer = Optim.minimizer(res)
+            @test typeof(_minimizer) <: Array{Float64, 3}
+            @test size(_minimizer) == (2,2,1)
+            if !(m in (SimulatedAnnealing, ParticleSwarm))
                 @test vecnorm(_minimizer - [10.0 0.0; 0.0 5.0]) < 10e-8
             end
         end