diff --git a/.gitignore b/.gitignore
index ded60678..01ff82ea 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,16 @@
*.py[cod]
+# Optimizers
+HPOlib/optimizers/smac_2_06_01-dev/*
+HPOlib/optimizers/spearmint_april2013_mod/*
+HPOlib/optimizers/hyperopt_august2013_mod/*
+
+# Runsolver
+runsolver/src/*
+
+# benchmark runs
+HPOlib/benchmarks/*/*_*/*
+
# C extensions
*.so
@@ -34,3 +45,9 @@ nosetests.xml
.mr.developer.cfg
.project
.pydevproject
+
+# pycharm
+.idea
+
+# Others
+*~
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 00000000..91252f78
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,16 @@
+language: python
+python:
+ - "2.7"
+# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
+
+before_install:
+ - sudo apt-get install -q libatlas3gf-base libatlas-dev liblapack-dev gfortran
+
+install:
+ - easy_install -U distribute
+ - pip install numpy
+ - pip install scipy==0.13.2
+ - pip install matplotlib
+ - python setup.py install
+# command to run tests, e.g. python setup.py test
+script: python setup.py test
diff --git a/Experiment.py b/HPOlib/Experiment.py
similarity index 83%
rename from Experiment.py
rename to HPOlib/Experiment.py
index 0f9cd3a2..0c64b64e 100644
--- a/Experiment.py
+++ b/HPOlib/Experiment.py
@@ -17,18 +17,26 @@
# along with this program. If not, see .
import cPickle
+import logging
import os
import scipy
+from scipy.stats.distributions import wrapcauchy_gen
import sys
import tempfile
import numpy as np
-import Locker
+import HPOlib.Locker as Locker
+import HPOlib.wrapping_util as wrapping_util
+
__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
__contact__ = "automl.org"
+
+logger = logging.getLogger("HPOlib.experiment")
+
+
CANDIDATE_STATE = 0
INCOMPLETE_STATE = 1
RUNNING_STATE = 2
@@ -48,10 +56,10 @@ def __init__(self, expt_dir, expt_name, max_wallclock_time=
self.locker = Locker.Locker()
# Only one process at a time is allowed to have access to this.
- sys.stderr.write("Waiting to lock experiments file " +
- self.jobs_pkl + "...")
+ #logger.info("Waiting to lock experiments file " +
+ # self.jobs_pkl + "...")
self.locker.lock_wait(self.jobs_pkl)
- sys.stderr.write("...acquired\n")
+ #logger.info("...acquired\n")
# Does this exist already?
if not os.path.exists(self.jobs_pkl):
@@ -118,7 +126,8 @@ def create_trial(self):
def __del__(self):
self._save_jobs()
if self.locker.unlock(self.jobs_pkl):
- sys.stderr.write("Released lock on job grid.\n")
+ pass
+ # sys.stderr.write("Released lock on job grid.\n")
else:
raise Exception("Could not release lock on job grid.\n")
@@ -158,20 +167,36 @@ def get_complete_jobs(self):
def get_broken_jobs(self):
return np.nonzero(self.status_array() == BROKEN_STATE)[0]
- # Get the best value so far
- def get_best(self):
- best = 0
+ # Get the job id of the best value so far, if there is no result
+ # available, this method consults the instance_results. If there are more
+ # than one trials with the same response value, the first trial is
+ # considered to be the best. If no trial with a better response value
+ # than sys.maxint is found, a ValueError is raised.
+ # TODO: add a method that incomplete jobs are not considered
+ def get_arg_best(self):
+ best_idx = -1
+ best_value = sys.maxint
for i, trial in enumerate(self.trials):
- res = np.NaN
- if trial['result'] == trial['result']:
- res = trial['result']
+ tmp_res = np.NaN
+ if np.isfinite(trial['result']):
+ tmp_res = trial['result']
elif np.isfinite(trial['instance_results']).any():
- res = scipy.nanmean(trial['instance_results'])
+ tmp_res = wrapping_util.nan_mean(trial['instance_results'])
+ # np.nanmean is not available in older numpy versions
+ # tmp_res = scipy.nanmean(trial['instance_results'])
else:
continue
- if res < self.trials[best]:
- best = i
- return self.trials[best]
+ if tmp_res < best_value:
+ best_idx = i
+ best_value = tmp_res
+ if best_idx == -1:
+ raise ValueError("No best value found.")
+ return best_idx
+
+ # Get the best value so far, for more documentation see get_arg_best
+ def get_best(self):
+ best_idx = self.get_arg_best()
+ return self.trials[best_idx]
def get_trial_from_id(self, _id):
return self.trials[_id]
@@ -203,8 +228,8 @@ def set_one_fold_crashed(self, _id, fold, result, duration):
self.trials[_id]['instance_status'][fold] = BROKEN_STATE
self.trials[_id]['instance_durations'][fold] = duration
self.trials[_id]['instance_results'][fold] = result
- if (self.get_trial_from_id(_id)['instance_status'] != RUNNING_STATE).all():
- self.get_trial_from_id(_id)['status'] = INCOMPLETE_STATE
+ if (self.trials[_id]['instance_status'] != RUNNING_STATE).all():
+ self.trials[_id]['status'] = INCOMPLETE_STATE
self.check_cv_finished(_id)
self.total_wallclock_time += duration
self._sanity_check()
@@ -218,8 +243,8 @@ def set_one_fold_complete(self, _id, fold, result, duration):
self.get_trial_from_id(_id)['instance_status'][fold] = COMPLETE_STATE
self.get_trial_from_id(_id)['instance_durations'][fold] = duration
# Set to incomplete if no job is running
- if (self.get_trial_from_id(_id)['instance_status'] != RUNNING_STATE).all():
- self.get_trial_from_id(_id)['status'] = INCOMPLETE_STATE
+ if (self.trials[_id]['instance_status'] != RUNNING_STATE).all():
+ self.trials[_id]['status'] = INCOMPLETE_STATE
# Check if all runs are finished
self.check_cv_finished(_id)
self.total_wallclock_time += duration
@@ -243,9 +268,13 @@ def check_cv_finished(self, _id):
self.get_trial_from_id(_id)['status'] = BROKEN_STATE
else:
self.get_trial_from_id(_id)['status'] = COMPLETE_STATE
- self.get_trial_from_id(_id)['result'] = np.sum(self.get_trial_from_id(_id)['instance_results']) / self.folds
- self.get_trial_from_id(_id)['std'] = np.std(self.get_trial_from_id(_id)['instance_results'])
- self.get_trial_from_id(_id)['duration'] = np.sum(self.get_trial_from_id(_id)['instance_durations'])
+ self.get_trial_from_id(_id)['result'] = \
+ np.sum(self.get_trial_from_id(_id)['instance_results'])\
+ / self.folds
+ self.get_trial_from_id(_id)['std'] =\
+ np.std(self.get_trial_from_id(_id)['instance_results'])
+ self.get_trial_from_id(_id)['duration'] =\
+ np.sum(self.get_trial_from_id(_id)['instance_durations'])
return True
else:
return False
@@ -254,13 +283,13 @@ def check_cv_finished(self, _id):
# parameters. Useful to delete all unnecessary entries after a crash in order
# to restart
def remove_all_but_first_runs(self, restored_runs):
- print "#########Restored runs", restored_runs
- print self.instance_order, len(self.instance_order)
+ logger.info("Restored runs %d", restored_runs)
+ logger.info("%s %s" ,self.instance_order, len(self.instance_order))
if len(self.instance_order) == restored_runs:
pass
else:
for _id, instance in self.instance_order[-1:restored_runs - 1:-1]:
- print "Deleting", _id, instance
+ logger.info("Deleting %d %d", _id, instance)
if np.isfinite(self.trials[_id]['instance_durations'][instance]):
self.total_wallclock_time -= \
self.trials[_id]['instance_durations'][instance]
@@ -326,12 +355,13 @@ def _sanity_check(self):
# Backwards compability with numpy 1.6
wallclock_time = np.nansum(trial['instance_durations'])
total_wallclock_time += wallclock_time if np.isfinite(wallclock_time) else 0
- assert (total_wallclock_time == self.total_wallclock_time), \
+ assert (wrapping_util.float_eq(total_wallclock_time,
+ self.total_wallclock_time)), \
(total_wallclock_time, self.total_wallclock_time)
# Automatically loads this object from a pickle file
def _load_jobs(self):
- fh = open(self.jobs_pkl, 'r')
+ fh = open(self.jobs_pkl, 'r')
jobs = cPickle.load(fh)
fh.close()
@@ -368,4 +398,4 @@ def _save_jobs(self):
'trials' : self.trials}, fh)
fh.close()
cmd = 'mv "%s" "%s"' % (fh.name, self.jobs_pkl)
- os.system(cmd) # TODO: Replace with subprocess modules
\ No newline at end of file
+ os.system(cmd) # TODO: Replace with subprocess modules
diff --git a/Locker.py b/HPOlib/Locker.py
similarity index 85%
rename from Locker.py
rename to HPOlib/Locker.py
index 69de10de..e80b198a 100755
--- a/Locker.py
+++ b/HPOlib/Locker.py
@@ -19,12 +19,16 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
+import logging
import os
-import sys
import time
+
+logger = logging.getLogger("HPOlib.locker")
+
+
def safe_delete(filename):
- cmd = 'mv "%s" "%s.delete" && rm "%s.delete"' % (filename, filename,
+ cmd = 'mv "%s" "%s.delete" && rm "%s.delete"' % (filename, filename,
filename)
fail = os.system(cmd)
return not fail
@@ -44,7 +48,7 @@ def lock(self, filename):
self.locks[filename] += 1
return True
else:
- cmd = 'ln -s /dev/null "%s.lock" 2> /dev/null' % (filename)
+ cmd = 'ln -s /dev/null "%s.lock" 2> /dev/null' % filename
fail = os.system(cmd)
if not fail:
self.locks[filename] = 1
@@ -52,13 +56,12 @@ def lock(self, filename):
def unlock(self, filename):
if not self.locks.has_key(filename):
- sys.stderr.write("Trying to unlock not-locked file %s.\n" %
- (filename))
+ logger.info("Trying to unlock not-locked file %s.\n", filename)
return True
if self.locks[filename] == 1:
success = safe_delete('%s.lock' % (filename))
if not success:
- sys.stderr.write("Could not unlock file: %s.\n" % (filename))
+ logger.log("Could not unlock file: %s.\n", filename)
del self.locks[filename]
return success
else:
diff --git a/HPOlib/Plotting/__init__.py b/HPOlib/Plotting/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/HPOlib/Plotting/__init__.py
@@ -0,0 +1 @@
+
diff --git a/HPOlib/Plotting/doAllPlots.py b/HPOlib/Plotting/doAllPlots.py
new file mode 100644
index 00000000..42132d2b
--- /dev/null
+++ b/HPOlib/Plotting/doAllPlots.py
@@ -0,0 +1,277 @@
+#!/usr/bin/env python
+
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+from argparse import ArgumentParser
+import os
+import subprocess
+import sys
+import time
+
+from HPOlib.Plotting import plot_util
+from HPOlib.wrapping_util import format_traceback
+
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+def _plot_trace(pkl_list, name_list, save="", cut=sys.maxint, log=False):
+ # We have one pkl per experiment
+
+ plotting_dir = os.path.dirname(os.path.realpath(__file__))
+ cur_dir = os.getcwd()
+
+ # noinspection PyBroadException
+ try:
+ os.chdir(plotting_dir)
+ import plotTrace
+ plotTrace.main(pkl_list, name_list, save=save, log=log, cut=cut)
+ os.chdir(cur_dir)
+ sys.stdout.write("passed\n")
+ except Exception, e:
+ sys.stderr.write(format_traceback(sys.exc_info()))
+ sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e))
+
+
+def _trace_with_std_per_eval(pkl_list, name_list, save="",
+ cut=sys.maxint, log=False):
+ plotting_dir = os.path.dirname(os.path.realpath(__file__))
+ cur_dir = os.getcwd()
+
+ # noinspection PyBroadException
+ try:
+ os.chdir(plotting_dir)
+ import plotTraceWithStd_perEval
+ plotTraceWithStd_perEval.main(pkl_list, name_list, autofill=True,
+ optimum=0, save=save, log=log, cut=cut)
+ os.chdir(cur_dir)
+ sys.stdout.write("passed\n")
+ except Exception, e:
+ sys.stderr.write(format_traceback(sys.exc_info()))
+ sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e))
+
+
+def _trace_with_std_per_time(pkl_list, name_list, save="",
+ cut=sys.maxint, log=False):
+ plotting_dir = os.path.dirname(os.path.realpath(__file__))
+ cur_dir = os.getcwd()
+
+ # noinspection PyBroadException
+ try:
+ os.chdir(plotting_dir)
+ import plotTraceWithStd_perTime
+ plotTraceWithStd_perTime.main(pkl_list, name_list, autofill=True,
+ optimum=0, save=save, log=log, cut=cut)
+ os.chdir(cur_dir)
+ sys.stdout.write("passed\n")
+ except Exception, e:
+ sys.stderr.write(format_traceback(sys.exc_info()))
+ sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e))
+
+
+def _optimizer_overhead(pkl_list, name_list, save="",
+ cut=sys.maxint, log=False):
+ plotting_dir = os.path.dirname(os.path.realpath(__file__))
+ cur_dir = os.getcwd()
+
+ # noinspection PyBroadException
+ try:
+ os.chdir(plotting_dir)
+ import plotOptimizerOverhead
+ plotOptimizerOverhead.main(pkl_list, name_list, autofill=True,
+ log=log, save=save, cut=cut)
+ os.chdir(cur_dir)
+ sys.stdout.write("passed\n")
+ except Exception, e:
+ sys.stderr.write(format_traceback(sys.exc_info()))
+ sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e))
+
+
+def _box_whisker(pkl_list, name_list, save="", cut=sys.maxint, log=False):
+ plotting_dir = os.path.dirname(os.path.realpath(__file__))
+ cur_dir = os.getcwd()
+
+ # noinspection PyBroadException
+ try:
+ os.chdir(plotting_dir)
+ import plotBoxWhisker
+ plotBoxWhisker.main(pkl_list, name_list, save=save, cut=cut)
+ os.chdir(cur_dir)
+ sys.stdout.write("passed\n")
+ except Exception, e:
+ sys.stderr.write(format_traceback(sys.exc_info()))
+ sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e))
+
+
+def _generate_tex_table(pkl_list, name_list, save="", cut=sys.maxint,
+ log=False):
+ plotting_dir = os.path.dirname(os.path.realpath(__file__))
+ cur_dir = os.getcwd()
+ # noinspection PyBroadException
+ try:
+ os.chdir(plotting_dir)
+ import generateTexTable
+ generateTexTable.main(pkl_list, name_list, save, cut)
+ os.chdir(cur_dir)
+ sys.stdout.write("passed\n")
+ except Exception, e:
+ sys.stderr.write(format_traceback(sys.exc_info()))
+ sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e))
+
+
+def _statistics(pkl_list, name_list, save="", cut=sys.maxint, log=False):
+ plotting_dir = os.path.dirname(os.path.realpath(__file__))
+ cur_dir = os.getcwd()
+
+ # noinspection PyBroadException
+ try:
+ os.chdir(plotting_dir)
+ cmd = ["python statistics.py", "--cut %d" % cut]
+ for i in range(len(name_list)):
+ cmd.append(name_list[i][0])
+ for pkl in pkl_list[i]:
+ cmd.append(pkl)
+ if save is not "":
+ fh = open(save, "w")
+ subprocess.check_call(" ".join(cmd), shell=True, stdin=fh, stdout=fh, stderr=fh)
+ fh.close()
+ else:
+ proc = subprocess.Popen(" ".join(cmd), shell=True, stdout=subprocess.PIPE)
+ out = proc.communicate()[0]
+ #print the output of the child process to stdout
+ print (out)
+ os.chdir(cur_dir)
+ sys.stdout.write("passed\n")
+ except Exception, e:
+ sys.stderr.write(format_traceback(sys.exc_info()))
+ sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e))
+
+
+def main():
+
+ prog = "python doAllPlots.py WhatIsThis [WhatIsThis ]"
+ description = "Tries to save as many plots as possible"
+
+ parser = ArgumentParser(description=description, prog=prog)
+
+ # General Options
+ parser.add_argument("-l", "--log", action="store_true", dest="log",
+ default=False, help="Plot on log scale")
+ parser.add_argument("-s", "--save", dest="save",
+ default="", help="Where to save plots? (directory)")
+ parser.add_argument("-f", "--file", dest="file",
+ default="png", help="File ending")
+ parser.add_argument("-c", "--cut", dest="cut", default=sys.maxint,
+ type=int, help="Cut experiment pickles after a specified number of trials.")
+
+ args, unknown = parser.parse_known_args()
+
+ sys.stdout.write("Found " + str(len(unknown)) + " arguments\n")
+
+ save_dir = os.path.realpath(args.save)
+
+ log = args.log
+
+ pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown)
+
+ if len(pkl_list) == 0:
+ raise ValueError("You must at least provide one experiment pickle.")
+
+ time_str = int(time.time() % 1000)
+
+ if not os.path.isdir(save_dir) and save_dir is not "":
+ os.mkdir(save_dir)
+
+ for i in range(len(pkl_list)):
+ for j in range(len(pkl_list[i])):
+ if os.path.exists(pkl_list[i][j]):
+ pkl_list[i][j] = os.path.abspath(pkl_list[i][j])
+ else:
+ raise NotImplementedError("%s is not a valid file" % pkl_list[i][j])
+
+ if len(name_list) == 1 and name_list[0][1] == 1:
+ # We have one exp and one pkl
+ if save_dir is not "":
+ tmp_save = os.path.join(save_dir, "plotTrace_%s.%s" % (time_str, args.file))
+ else:
+ tmp_save = save_dir
+ sys.stdout.write("plotTrace.py ... %s ..." % tmp_save)
+ _plot_trace(pkl_list=pkl_list, name_list=name_list, save=tmp_save,
+ log=log, cut=args.cut)
+
+ if len(name_list) > 1:
+ # Some plots only make sense, if there are many experiments
+ # BoxWhisker
+ if save_dir is not "":
+ tmp_save = os.path.join(save_dir, "BoxWhisker_%s.%s" % (time_str, args.file))
+ else:
+ tmp_save = save_dir
+ sys.stdout.write("plotBoxWhisker.py ... %s ..." % tmp_save)
+ _box_whisker(pkl_list=pkl_list, name_list=name_list, save=tmp_save,
+ log=log, cut=args.cut)
+
+ # statistics
+ if save_dir is not "":
+ tmp_save = os.path.join(save_dir, "statistics_%s.txt" % time_str)
+ else:
+ tmp_save = save_dir
+ sys.stdout.write("statistics.py ... %s ..." % tmp_save)
+ _statistics(pkl_list=pkl_list, name_list=name_list, save=tmp_save,
+ log=log, cut=args.cut)
+
+ # LaTeX table
+ if save_dir is not "":
+ tmp_save = os.path.join(save_dir, "table_%s.tex" % time_str)
+ else:
+ tmp_save = save_dir
+ sys.stdout.write("generateTexTable.py ... %s ..." % tmp_save)
+ _generate_tex_table(pkl_list=pkl_list, name_list=name_list,
+ save=tmp_save, log=log, cut=args.cut)
+
+
+ # We can always plot this
+ # OptimizerOverhead
+ if save_dir is not "":
+ tmp_save = os.path.join(save_dir, "OptimizerOverhead_%s.%s" % (time_str, args.file))
+ else:
+ tmp_save = save_dir
+ sys.stdout.write("plotOptimizerOverhead.py ... %s ..." % tmp_save)
+ _optimizer_overhead(pkl_list=pkl_list, name_list=name_list, save=tmp_save,
+ log=log, cut=args.cut)
+
+ # Error Trace with Std
+ if save_dir is not "":
+ tmp_save = os.path.join(save_dir, "TraceWithStd_perEval_%s.%s" % (time_str, args.file))
+ else:
+ tmp_save = save_dir
+ sys.stdout.write("TraceWithStd_perEval.py ... %s ..." % tmp_save)
+ _trace_with_std_per_eval(pkl_list=pkl_list, name_list=name_list,
+ save=tmp_save, log=log, cut=args.cut)
+
+ if save_dir is not "":
+ tmp_save = os.path.join(save_dir, "TraceWithStd_perTime_%s.%s" % (time_str, args.file))
+ else:
+ tmp_save = save_dir
+ sys.stdout.write("TraceWithStd_perTime.py ... %s ..." % tmp_save)
+ _trace_with_std_per_time(pkl_list=pkl_list, name_list=name_list,
+ save=tmp_save, log=log, cut=args.cut)
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/HPOlib/Plotting/generateTexTable.py b/HPOlib/Plotting/generateTexTable.py
new file mode 100644
index 00000000..76cff0ec
--- /dev/null
+++ b/HPOlib/Plotting/generateTexTable.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+from argparse import ArgumentParser
+from collections import OrderedDict
+from StringIO import StringIO
+import numpy as np
+import sys
+
+try:
+ import jinja2
+ from jinja2 import Template
+except ImportError:
+ jinja2 = ""
+
+from HPOlib.Plotting import plot_util
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+template_string = \
+"""\\documentclass[landscape]{article} % For LaTeX2
+
+\\usepackage[landscape]{geometry}
+\\usepackage{multirow} % import command \multicolmun
+\\usepackage{tabularx} % Convenient table formatting
+\\usepackage{booktabs} % provides \toprule, \midrule and \bottomrule
+
+\\begin{document}
+
+\\begin{table}[t]
+\\begin{tabularx}{\\textwidth}{lr{%- for name in result_values -%}|Xr{%- endfor -%}}
+\\toprule
+\multicolumn{2}{l}{}
+{%- for name in result_values -%}
+&\multicolumn{2}{c}{\\bf {{name}}}
+{%- endfor -%}
+\\\\
+\\multicolumn{1}{l}{\\bf Experiment} &\multicolumn{1}{r}{\\#evals}
+{%- for name in result_values -%}
+&\\multicolumn{1}{l}{Valid.\\ loss} &\\multicolumn{1}{r}{Best loss}
+{%- endfor -%}
+\\\\
+\\toprule
+{{ experiment }} & {{ evals }}
+{%- for name in result_values -%}
+{%- set results = result_values[name] -%}
+{{ ' & ' }}{{ results['mean']|round(3, 'floor') }}$\\pm${{ results['std']|round(3, 'floor')}} & {{ results['min']|round(3, 'floor') }}
+{%- endfor %} \\\\
+\\bottomrule
+\\end{tabularx}
+\\end{table}
+\\end{document}
+"""
+
+def main(pkl_list, name_list, save, cut=sys.maxint):
+ tex = StringIO()
+ result_values = OrderedDict([(name[0], dict()) for name in name_list])
+
+ best_dict, idx_dict, keys = plot_util.read_pickles(name_list, pkl_list, cut)
+
+ for name in result_values:
+ values = result_values[name]
+
+ values["mean"] = np.mean(best_dict[name])
+ values["mean_bold"] = False
+ values["mean_italic"] = False
+
+ values["std"] = np.std(best_dict[name])
+ values["std_bold"] = False
+ values["std_italic"] = False
+
+ values["min"] = np.min(best_dict[name])
+ values["min_bold"] = False
+ values["min_italic"] = False
+
+ values["max"] = np.min(best_dict[name])
+ values["max_bold"] = False
+ values["max_italic"] = False
+
+ if jinja2:
+ template = Template(template_string)
+ tex.write(template.render(result_values=result_values,
+ experiment="Name", evals="\\#evals"))
+ else:
+ tex.write("Name & #evals")
+ for name in result_values:
+ values = result_values[name]
+ tex.write(" & ")
+ tex.write(values["mean"])
+ tex.write("$\\pm$")
+ tex.write(values["std"])
+ tex.write(" & ")
+ tex.write(values["min"])
+ tex.write("\\\\")
+
+ tex.seek(0)
+ table = tex.getvalue()
+
+ if save != "":
+ with open(save, "w") as fh:
+ fh.write(table)
+ else:
+ print table
+
+
+if __name__ == "__main__":
+ prog = "python statistics.py WhatIsThis WhatIsThis [WhatIsThis ]"
+ description = "Generate a LaTeX results table."
+
+ parser = ArgumentParser(description=description, prog=prog)
+
+ parser.add_argument("-c", "--cut", dest="cut", default=sys.maxint,
+ type=int, help="Only consider that many evaluations")
+ parser.add_argument("-s", "--save", dest="save", default="",
+ help="Where to save plot instead of showing it?")
+ args, unknown = parser.parse_known_args()
+ # TODO-list:
+ # 1. Add statistical relevance
+ # 2. Add parameters to control whether a value should be printed bold
+ # face, underlined or whatever
+ # 3. Add a custom rounding
+ # 4. Determine the experiment name and number of evaluations
+
+ pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown)
+ main(pkl_list_main, name_list_main, args.save, args.cut)
+
+
diff --git a/HPOlib/Plotting/plotBoxWhisker.py b/HPOlib/Plotting/plotBoxWhisker.py
new file mode 100644
index 00000000..aaa75f85
--- /dev/null
+++ b/HPOlib/Plotting/plotBoxWhisker.py
@@ -0,0 +1,151 @@
+#!/usr/bin/env python
+
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+from argparse import ArgumentParser
+
+import cPickle
+import sys
+
+from matplotlib.gridspec import GridSpec
+from matplotlib.pyplot import xticks, figure, subplot, savefig, show, tight_layout, subplots_adjust
+
+import numpy as np
+
+from HPOlib.Plotting import plot_util
+
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__license__ = "3-clause BSD License"
+__contact__ = "automl.org"
+
+
+def plot_box_whisker(best_trials, name_list, title="", save="", y_min=0,
+ y_max=0):
+ ratio = 5
+ gs = GridSpec(ratio, 1)
+ fig = figure(1, dpi=100)
+ fig.suptitle(title)
+ ax = subplot(gs[0:ratio, :])
+ ax.yaxis.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
+
+ bp = ax.boxplot(best_trials, 0, 'ok')
+ boxlines = bp['boxes']
+ for line in boxlines:
+ line.set_color('k')
+ line.set_linewidth(2)
+
+ min_y = sys.maxint
+ max_y = -sys.maxint
+
+ # Get medians and limits
+ medians = range(len(name_list))
+ for i in range(len(name_list)):
+ med = bp['medians'][i]
+ median_x = []
+ median_y = []
+ for j in range(2):
+ median_x.append(med.get_xdata()[j])
+ median_y.append(med.get_ydata()[j])
+ medians[i] = median_y[0]
+ if min(best_trials[i]) < min_y:
+ min_y = min(best_trials[i])
+ if max(best_trials[i]) > max_y:
+ max_y = max(best_trials[i])
+ print medians
+
+ # Plot xticks
+ xticks(range(1, len(name_list)+1), name_list)
+
+ # Set limits
+ if y_max == y_min:
+ # Set axes limit
+ ax.set_ylim([min_y-0.1*abs((max_y-min_y)), max_y+0.1*abs((max_y-min_y))])
+ else:
+ ax.set_ylim([y_min, y_max])
+ max_y = y_max
+ min_y = y_min
+
+ # Print medians as upper labels
+ top = max_y-((max_y-min_y)*0.05)
+ pos = np.arange(len(name_list))+1
+ upper_labels = [str(np.round(s, 5)) for s in medians]
+ upper_labels[0] = "median=[%s," % upper_labels[0]
+ for i in range(len(upper_labels[1:-1])):
+ upper_labels[i+1] = "%s," % upper_labels[i+1]
+ upper_labels[-1] = "%s]" % upper_labels[-1]
+ for tick, label in zip(range(len(name_list)), ax.get_xticklabels()):
+ ax.text(pos[tick], top, upper_labels[tick],
+ horizontalalignment='center', size='x-small')
+
+ ax.set_ylabel('Minfunction')
+
+ tight_layout()
+ subplots_adjust(top=0.85)
+ if save != "":
+ savefig(save, dpi=100, facecolor='w', edgecolor='w',
+ orientation='portrait', papertype=None, format=None,
+ transparent=False, bbox_inches="tight", pad_inches=0.1)
+ else:
+ show()
+
+
+def main(pkl_list, name_list, title="", save="", y_min=0, y_max=0, cut=sys.maxint):
+
+ best_trials = list()
+ for i in range(len(name_list)):
+ best_trials.append(list())
+ for pkl in pkl_list[i]:
+ fh = open(pkl, "r")
+ trials = cPickle.load(fh)
+ fh.close()
+ best_trials[i].append(plot_util.get_best(trials, cut=cut))
+
+ plot_box_whisker(best_trials=best_trials, name_list=name_list,
+ title=title, save=save, y_min=y_min, y_max=y_max)
+
+ if save != "":
+ sys.stdout.write("Saving plot to " + save + "\n")
+ else:
+ sys.stdout.write("..Done\n")
+
+if __name__ == "__main__":
+ prog = "python plotBoxWhisker.py WhatIsThis [WhatIsThis ]"
+ description = "Plot a Box whisker plot for many experiments. The box covers lower to upper quartile."
+
+ parser = ArgumentParser(description=description, prog=prog)
+
+ # General Options
+ parser.add_argument("-t", "--title", dest="title", default="",
+ help="Optional supertitle for plot")
+ parser.add_argument("--max", dest="max", default=0,
+ type=float, help="Maximum of the plot")
+ parser.add_argument("--min", dest="min", default=0,
+ type=float, help="Minimum of the plot")
+ parser.add_argument("-s", "--save", dest="save", default="",
+ help="Where to save plot instead of showing it?")
+ parser.add_argument("-c", "--cut", default=sys.maxint, type=int,
+ help="Cut experiment pickles after a specified number of trials.")
+ args, unknown = parser.parse_known_args()
+
+ sys.stdout.write("\nFound " + str(len(unknown)) + " arguments...")
+
+ pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown)
+
+ main(pkl_list=pkl_list_main, name_list=name_list_main, title=args.title, save=args.save,
+ y_min=args.min, y_max=args.max, cut=args.cut)
diff --git a/HPOlib/Plotting/plotBranin.py b/HPOlib/Plotting/plotBranin.py
new file mode 100644
index 00000000..b5a7fb49
--- /dev/null
+++ b/HPOlib/Plotting/plotBranin.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+from argparse import ArgumentParser
+
+import cPickle
+import itertools
+import sys
+
+import matplotlib.cm
+import matplotlib.gridspec as gridSpec
+import matplotlib.pyplot
+import numpy as np
+
+import HPOlib.benchmark_functions
+import HPOlib.Plotting.plot_util as plotUtil
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+def plot_contour(trial_list, name_list, save="", title=""):
+ # constraints:
+ # -5 <= x <= 10, 0 <= y <= 15
+ # three global optima: (-pi, 12.275), (pi, 2.275), (9.42478, 2.475), where
+ # branin = 0.397887
+
+ markers = itertools.cycle(['o', 's', '^', 'x'])
+ colors = itertools.cycle(['b', 'g', 'r', 'k'])
+ size = 5
+
+ # Get handles
+ ratio = 5
+ gs = gridSpec.GridSpec(ratio, 1)
+ fig = matplotlib.pyplot.figure(1, dpi=100)
+ fig.suptitle(title)
+ ax = matplotlib.pyplot.subplot(gs[0:ratio, :])
+ ax.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
+ xopt = [-np.pi, np.pi, 9.42478]
+ yopt = [12.275, 2.275, 2.475]
+
+ # Plot Branin
+ step = 0.1
+ xi = np.arange(-5, 10 + step, step)
+ yi = np.arange(-0, 15 + step, step)
+
+ z = np.zeros([len(xi), len(yi)])
+ for i in range(len(xi)):
+ for j in range(len(yi)):
+ #z[j, i] = np.power(np.e, branin.branin({"x":xi[i], "y":yi[j]}))
+ z[j, i] = HPOlib.benchmark_functions.branin(x=xi[i], y=yi[j])
+ xi, yi = np.meshgrid(xi, yi)
+ cax = ax.contourf(xi, yi, z, 50, cmap=matplotlib.cm.gray)
+ fig.colorbar(cax)
+
+ # Plot Optimums after all work is done
+ matplotlib.pyplot.scatter(xopt, yopt, marker="o", facecolor='w', edgecolor='w', s=20*size, label="Optimum")
+
+ # Get values
+ for opt in range(len(name_list)):
+ print name_list[opt], "has", len(trial_list[opt]['trials']), "samples"
+ m = markers.next()
+ c = colors.next()
+ x = np.zeros(len(trial_list[opt]["trials"]))
+ y = np.zeros(len(trial_list[opt]["trials"]))
+ for i in range(len(x)):
+ if '-x' in trial_list[opt]["trials"][i]["params"]:
+ x[i] = float(trial_list[opt]["trials"][i]["params"]["-x"].strip("'"))
+ y[i] = float(trial_list[opt]["trials"][i]["params"]["-y"].strip("'"))
+ else:
+ x[i] = float(trial_list[opt]["trials"][i]["params"]["x"].strip("'"))
+ y[i] = float(trial_list[opt]["trials"][i]["params"]["y"].strip("'"))
+ matplotlib.pyplot.scatter(x[0:10], y[0:10], marker=m,
+ s=size, facecolors=c, linewidth=0.1)
+ matplotlib.pyplot.scatter(x[10:-10], y[10:-10], marker=m,
+ linewidth=0.1, s=4*size, facecolors=c)
+ matplotlib.pyplot.scatter(x[-10:-1], y[-10:-1], marker=m,
+ linewidth=0.1, s=6*size, facecolors=c, label=name_list[opt][0])
+
+ matplotlib.pyplot.xlim([-5, 10])
+ matplotlib.pyplot.ylim([-0, 15])
+ matplotlib.pyplot.xlabel("X")
+ matplotlib.pyplot.ylabel("Y")
+
+ # Describe the plot
+ matplotlib.pyplot.title(title)
+ leg = matplotlib.pyplot.legend(loc="best", fancybox=True)
+ leg.get_frame().set_alpha(0.5)
+
+ if save != "":
+ matplotlib.pyplot.subplots_adjust(top=0.85)
+ matplotlib.pyplot.savefig(save, dpi=600, facecolor='w', edgecolor='w',
+ orientation='portrait', papertype=None, format=None,
+ transparent=False, bbox_inches="tight", pad_inches=0.1)
+ else:
+ matplotlib.pyplot.show()
+
+
+def main():
+ prog = "python plotBranin.py whatIsThis [whatIsThis] ]"
+ description = "Plot a Trace with std for multiple experiments"
+
+ parser = ArgumentParser(description=description, prog=prog)
+
+ parser.add_argument("-s", "--save", dest="save", default="",
+ help="Where to save plot instead of showing it?")
+ parser.add_argument("-t", "--title", dest="title", default="",
+ help="Optional supertitle for plot")
+
+ args, unknown = parser.parse_known_args()
+
+ if len(unknown) % 2 != 0:
+ print "Wrong number of arguments", len(args)
+ print prog
+ sys.exit(1)
+
+ pkl_list, name_list = plotUtil.get_pkl_and_name_list(unknown)
+ trial_list = list()
+ for i in range(len(name_list)):
+ result_file = pkl_list[i][0]
+ fh = open(result_file, "r")
+ trials = cPickle.load(fh)
+ fh.close()
+ trial_list.append(trials)
+
+ plot_contour(trial_list=trial_list, name_list=name_list, save=args.save, title=args.title)
+
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/HPOlib/Plotting/plotOptimizerOverhead.py b/HPOlib/Plotting/plotOptimizerOverhead.py
new file mode 100644
index 00000000..35c3538f
--- /dev/null
+++ b/HPOlib/Plotting/plotOptimizerOverhead.py
@@ -0,0 +1,204 @@
+#!/usr/bin/env python
+
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+from argparse import ArgumentParser
+
+import cPickle
+import itertools
+import sys
+
+import matplotlib.pyplot as plt
+import matplotlib.gridspec
+import numpy as np
+
+from HPOlib.Plotting import plot_util
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+def plot_time_trace(time_dict, name_list, title="", log=True, save="", y_max=0, y_min=0):
+ colors = plot_util.get_plot_colors()
+ markers = plot_util.get_plot_markers()
+ linestyles = itertools.cycle(['-'])
+
+ size = 5
+ ratio = 5
+ gs = matplotlib.gridspec.GridSpec(ratio, 1)
+ fig = plt.figure(1, dpi=100)
+ fig.suptitle(title, fontsize=16)
+ ax1 = plt.subplot(gs[0:ratio, :])
+ ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
+ min_val = sys.maxint
+ max_val = -sys.maxint
+ max_trials = 0
+
+ trial_list_means = list()
+ trial_list_std = list()
+ num_runs_list = list()
+
+ # Get mean and std for all times and optimizers
+ for entry in name_list:
+ k = entry[0]
+ trial_list_std.append(np.std(np.array(time_dict[k]), axis=0))
+ if log:
+ trial_list_means.append(np.log10(np.mean(np.array(time_dict[k]), axis=0)))
+ else:
+ trial_list_means.append(np.mean(np.array(time_dict[k]), axis=0))
+ num_runs_list.append(len(time_dict[k]))
+
+ for k in range(len(name_list)):
+ # Plot mean and std for optimizer duration
+ c = colors.next()
+ m = markers.next()
+ x = range(len(trial_list_means[k]))
+ l = linestyles.next()
+ ax1.fill_between(x, trial_list_means[k] - trial_list_std[k],
+ trial_list_means[k] + trial_list_std[k],
+ facecolor=c, alpha=0.3, edgecolor=c)
+ ax1.plot(x, trial_list_means[k], color=c, linewidth=size, label=name_list[k][0], linestyle=l, marker=m)
+ # Plot number of func evals for this experiment
+
+ if min(trial_list_means[k] - trial_list_std[k]) < min_val:
+ min_val = min(trial_list_means[k] - trial_list_std[k])
+ if max(trial_list_means[k] + trial_list_std[k]) > max_val:
+ max_val = max(trial_list_means[k] + trial_list_std[k])
+ if len(trial_list_means[k]) > max_trials:
+ max_trials = len(trial_list_means[k])
+
+ # Descript and label the stuff
+ fig.suptitle(title, fontsize=16)
+ leg = ax1.legend(loc='best', fancybox=True)
+ leg.get_frame().set_alpha(0.5)
+ if log:
+ ax1.set_ylabel("log10(Optimizer time in [sec])")
+ else:
+ ax1.set_ylabel("Optimizer time in [sec]")
+ if y_max == y_min:
+ ax1.set_ylim([min_val-2, max_val+2])
+ else:
+ ax1.set_ylim([y_min, y_max])
+ ax1.set_xlim([0, max_trials])
+
+ plt.tight_layout()
+ plt.subplots_adjust(top=0.85)
+ if save != "":
+ plt.savefig(save, dpi=100, facecolor='w', edgecolor='w',
+ orientation='portrait', papertype=None, format=None,
+ transparent=False, bbox_inches="tight", pad_inches=0.1)
+ else:
+ plt.show()
+
+
+def main(pkl_list, name_list, autofill, title="", log=False, save="",
+ y_min=0, y_max=0, cut=sys.maxint):
+
+ times_dict = dict()
+ for exp in range(len(name_list)):
+ times_dict[name_list[exp][0]] = list()
+ for pkl in pkl_list[exp]:
+ fh = open(pkl, "r")
+ trials = cPickle.load(fh)
+ fh.close()
+
+ # Get all variables from the trials object
+ cv_starttime = trials["cv_starttime"][:cut]
+ cv_endtime = trials["cv_endtime"][:cut]
+
+ # Get optimizer duration times
+ time_list = list()
+ # First duration
+ time_list.append(cv_starttime[0] - trials["starttime"][0])
+ time_idx = 0
+ for i in range(len(cv_starttime[1:])):
+ # Is there a next restored run?
+ # if yes, does next cvstart belong to a restored run?
+ if len(trials["endtime"]) > time_idx and \
+ cv_starttime[i+1] > trials["endtime"][time_idx]:
+ # Check whether run crashed/terminated during cv
+ # Equals means that the run crashed
+ if cv_endtime[i] < trials["endtime"][time_idx]:
+ # No .. everything is fine
+ time_list.append((trials["endtime"][time_idx] - cv_endtime[i]))
+ time_list.append((cv_starttime[i + 1] - trials["starttime"][time_idx+1]))
+ elif trials["endtime"][time_idx] == cv_endtime[i]:
+ # Yes, but BBoM managed to set an endtime
+ pass
+ else:
+ # Yes ... trouble
+ print "Help"
+ print trials["endtime"][time_idx]
+ print cv_endtime[i]
+ time_idx += 1
+ # everything ...
+ else:
+ time_list.append(cv_starttime[i + 1] - cv_endtime[i])
+ times_dict[name_list[exp][0]].append(time_list)
+
+ for key in times_dict.keys():
+ max_len = max([len(ls) for ls in times_dict[key]])
+ for t in range(len(times_dict[key])):
+ if len(times_dict[key][t]) < max_len and autofill:
+ diff = max_len - len(times_dict[key][t])
+ # noinspection PyUnusedLocal
+ times_dict[key][t] = np.append(times_dict[key][t], [times_dict[key][t][-1] for x in range(diff)])
+ elif len(times_dict[key][t]) < max_len and not autofill:
+ raise ValueError("(%s != %s), Traces do not have the same length, please use -a" %
+ (str(max_len), str(len(times_dict[key][t]))))
+
+ plot_time_trace(times_dict, name_list, title=title, log=log, save=save,
+ y_min=y_min, y_max=y_max)
+
+ if save != "":
+ sys.stdout.write("Saved plot to " + save + "\n")
+ else:
+ sys.stdout.write("..Done\n")
+
+if __name__ == "__main__":
+ prog = "python plotOptimizerOverhead.py WhatIsThis [WhatIsThis ]"
+ description = "Plot a Trace with std for multiple experiments"
+
+ parser = ArgumentParser(description=description, prog=prog)
+
+ # General Options
+ parser.add_argument("-c", "--cut", type=int, default=sys.maxint,
+ help="Cut the experiment pickle length.")
+ parser.add_argument("-l", "--log", action="store_true", dest="log",
+ default=False, help="Plot on log scale")
+ parser.add_argument("--max", type=float, dest="max",
+ default=0, help="Maximum of the plot")
+ parser.add_argument("--min", type=float, dest="min",
+ default=0, help="Minimum of the plot")
+ parser.add_argument("-s", "--save", dest="save",
+ default="", help="Where to save plot instead of showing it?")
+ parser.add_argument("-t", "--title", dest="title",
+ default="", help="Choose a supertitle for the plot")
+
+ # Options which are available only for this plot
+ parser.add_argument("-a", "--autofill", action="store_true", dest="autofill",
+ default=False, help="Fill trace automatically")
+
+ args, unknown = parser.parse_known_args()
+
+ sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n")
+
+ pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown)
+ main(pkl_list_main, name_list_main, autofill=args.autofill, title=args.title,
+ log=args.log, save=args.save, y_min=args.min, y_max=args.max,
+ cut=args.cut)
diff --git a/HPOlib/Plotting/plotParam.py b/HPOlib/Plotting/plotParam.py
new file mode 100644
index 00000000..d660cb47
--- /dev/null
+++ b/HPOlib/Plotting/plotParam.py
@@ -0,0 +1,181 @@
+#!/usr/bin/env python
+
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+from argparse import ArgumentParser
+
+import cPickle
+import re
+import sys
+
+from matplotlib.pyplot import tight_layout, figure, subplots_adjust, subplot, savefig, show
+from matplotlib.gridspec import GridSpec
+
+import numpy as np
+
+from HPOlib.Plotting import plot_util
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+def translate_para(key, value):
+ # sanitize all params
+ new_name = key
+ if "LOG10" in key:
+ pos = key.find("LOG10")
+ new_name = key[0:pos] + key[pos+5:]
+ new_name = new_name.strip("_")
+ value = np.power(10, float(value))
+ elif "LOG2" in key:
+ pos = key.find("LOG2")
+ new_name = key[0:pos] + key[pos+4:]
+ new_name = new_name.strip("_")
+ value = np.power(2, float(value))
+ elif "LOG" in key:
+ pos = key.find("LOG")
+ new_name = key[0:pos] + key[pos+3:]
+ new_name = new_name.strip("_")
+ value = np.exp(float(value))
+ #Check for Q value, returns round(x/q)*q
+ m = re.search('Q[0-999]{1,3}', key)
+ if m is not None:
+ pos = new_name.find(m.group(0))
+ tmp = new_name[0:pos] + new_name[pos+3:]
+ new_name = tmp.strip("_")
+ q = float(m.group(0)[1:])
+ value = round(float(value)/q)*q
+ return new_name, value
+
+
+def plot_params(value_list, result_list, name, save="", title="", jitter=0):
+ color = 'k'
+ marker = 'o'
+ size = 1
+
+ # Get handles
+ ratio = 5
+ gs = GridSpec(ratio, 4)
+ fig = figure(1, dpi=100)
+ fig.suptitle(title, fontsize=16)
+ ax = subplot(gs[:, :])
+ ax.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
+
+ # Define xlims
+ min_y = min(result_list)
+ max_y = max(result_list)
+ y_offset = np.abs(max_y - min_y) * 0.1
+ min_y -= y_offset
+ max_y += y_offset
+
+ min_x = min(value_list)
+ max_x = max(value_list)
+ x_offset = np.abs(max_x - min_x) * 0.1
+ min_x -= x_offset
+ max_x += x_offset
+
+ # Maybe jitter data
+ # But before save best values
+ best_value = value_list[np.argmin(result_list)]
+ best_result = min(result_list)
+
+ for idx in range(len(result_list)):
+ # noinspection PyArgumentList
+ result_list[idx] += float((np.random.rand(1) - 0.5) * jitter)
+ # noinspection PyArgumentList
+ value_list[idx] += float((np.random.rand(1) - 0.5) * jitter)
+
+ # Plot
+ ax.scatter(value_list, result_list, facecolor=color, edgecolor=color, marker=marker, s=size*50)
+ ax.scatter(best_value, best_result, facecolor='r',
+ edgecolor='r', s=size*150, marker='o', alpha=0.5,
+ label="f[%5.3f]=%5.3f" % (best_value, best_result))
+
+ # Describe and label the stuff
+ ax.set_xlabel("Value of %s, jitter=%f" % (name, jitter))
+ ax.set_ylabel("Minfunction value, jitter=%f" % jitter)
+
+ leg = ax.legend(loc='best', fancybox=True, scatterpoints=1)
+ leg.get_frame().set_alpha(0.5)
+
+ # Apply the xlims
+ ax.set_xlim([min_x, max_x])
+ ax.set_ylim([min_y, max_y])
+
+ # Save Plot
+ tight_layout()
+ subplots_adjust(top=0.85)
+ if save != "":
+ savefig(save, dpi=100, facecolor='w', edgecolor='w',
+ orientation='portrait', papertype=None, format=None,
+ transparent=False, bbox_inches="tight", pad_inches=0.1)
+ else:
+ show()
+
+
+def main(pkl_list, name_list, param, save="", title="", jitter=0):
+ if len(name_list) > 1:
+ raise NotImplementedError("No support for more than one experiment")
+
+ mod_param = "-" + param
+
+ value_list = list()
+ result_list = list()
+ param_set = set()
+ for pkl in pkl_list[0]:
+ fh = open(pkl, "r")
+ trials = cPickle.load(fh)
+ fh.close()
+ for t in trials["trials"]:
+ if mod_param in t["params"]:
+ k, value = translate_para(param, t["params"][mod_param])
+ value_list.append(float(value.strip("'")))
+ result_list.append(t["result"])
+ param_set.update(t["params"].keys())
+
+ if len(value_list) == 0:
+ print("No values found for param '%s', Available params:\n%s" %
+ (param, "\n".join([p[1:] for p in param_set])))
+ sys.exit(1)
+ else:
+ print "Found %s values for %s" % (str(len(value_list)), param)
+
+ plot_params(value_list=value_list, result_list=result_list, name=param, save=save, title=title,
+ jitter=jitter)
+
+if __name__ == "__main__":
+ prog = "python plot_param.py WhatIsThis * "
+ description = "Plot one param wrt to minfunction value"
+
+ parser = ArgumentParser(description=description, prog=prog)
+ parser.add_argument("-s", "--save", dest="save", default="",
+ help="Where to save plot instead of showing it?")
+ parser.add_argument("-p", "--parameter", dest="param", default="",
+ help="Which parameter to plot")
+ parser.add_argument("-j", "--jitter", dest="jitter", default=0, type=float,
+ help="Jitter data?")
+ parser.add_argument("-t", "--title", dest="title", default="",
+ help="Choose a supertitle for the plot")
+
+ args, unknown = parser.parse_known_args()
+
+ sys.stdout.write("\nFound " + str(len(unknown)) + " argument[s]...\n")
+ pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown)
+ main(pkl_list=pkl_list_main, name_list=name_list_main, param=args.param,
+ save=args.save, title=args.title, jitter=args.jitter)
diff --git a/HPOlib/Plotting/plotTrace.py b/HPOlib/Plotting/plotTrace.py
new file mode 100644
index 00000000..b5cf3b9d
--- /dev/null
+++ b/HPOlib/Plotting/plotTrace.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python
+
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+from argparse import ArgumentParser
+import itertools
+import sys
+import cPickle
+
+from matplotlib.pyplot import tight_layout, figure, subplots_adjust, subplot, savefig, show, yscale
+import matplotlib.gridspec
+import numpy as np
+import scipy.stats as sc
+
+from HPOlib.Plotting import plot_util
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+def plot_optimization_trace(trial_list, name_list, optimum=0, title="", log=False,
+ save="", y_min=0, y_max=0, cut=sys.maxint):
+ markers = plot_util.get_plot_markers()
+ colors = plot_util.get_plot_colors()
+ linestyles = itertools.cycle(['-'])
+ size = 1
+
+ # get handles
+ ratio = 5
+ gs = matplotlib.gridspec.GridSpec(ratio, 1)
+ fig = figure(1, dpi=100)
+ fig.suptitle(title, fontsize=16)
+ ax = subplot(gs[0:ratio, :])
+ ax.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
+ min_val = sys.maxint
+ max_val = -sys.maxint
+ max_trials = 0
+
+ # This might not do what we actually want; Ideally it would only take the
+ # ones that where actually run according to instance_order
+ for i in range(len(name_list)):
+ print cut, len(trial_list[i])
+ num_plotted_trials = np.min([cut, len(trial_list[i])])
+ print num_plotted_trials
+ x = range(num_plotted_trials)
+ y = np.zeros((num_plotted_trials))
+ line = np.zeros((num_plotted_trials))
+ for j, inst_res in enumerate(trial_list[i][:num_plotted_trials]):
+ if j >= len(y):
+ break
+ if type(inst_res) == np.ndarray and not np.isfinite(inst_res).any():
+ inst_res[np.isnan(inst_res)] = 1
+ elif type(inst_res) != np.ndarray and np.isnan(inst_res):
+ inst_res = 1
+ tmp = sc.nanmean(np.array([inst_res, inst_res]).flat) # Black Magic
+ if log:
+ y[j] = np.log(tmp - optimum)
+ line[j] = np.min(y[:j + 1])
+ else:
+ y[j] = tmp - optimum
+ line[j] = np.min(y[:j + 1])
+
+ # Plot the stuff
+ marker = markers.next()
+ color = colors.next()
+ l = linestyles.next()
+ ax.scatter(np.argmin(line), min(line), facecolor="w", edgecolor=color,
+ s=size*10*15, marker=marker)
+ ax.scatter(x, y, color=color, marker=marker, s=size*15)
+ ax.plot(x, line, color=color, label=name_list[i][0], linestyle=l, linewidth=size)
+
+ if min(y) < min_val:
+ min_val = min(y)
+ if max(y) > max_val:
+ max_val = max(y)
+ if num_plotted_trials > max_trials:
+ max_trials = num_plotted_trials
+
+ # Describe and label the stuff
+ ax.set_xlabel("#Function evaluations")
+ if log:
+ ax.set_ylabel("log10(Minfunction value)")
+ else:
+ ax.set_ylabel("Minfunction value")
+
+ if y_min == y_max:
+ ax.set_ylim([min_val - 0.1, max_val + 0.1])
+ else:
+ ax.set_ylim([y_min, y_max])
+ ax.set_xlim([0, max_trials])
+
+ leg = ax.legend(loc='best', fancybox=True)
+ leg.get_frame().set_alpha(0.5)
+
+ tight_layout()
+ subplots_adjust(top=0.85)
+
+ if save != "":
+ savefig(save, dpi=100, facecolor='w', edgecolor='w',
+ orientation='portrait', papertype=None, format=None,
+ transparent=False, bbox_inches="tight", pad_inches=0.1)
+ else:
+ show()
+
+
+def main(pkl_list, name_list, optimum=0, title="", log=False, save="", y_max=0,
+ y_min=0, cut=sys.maxint):
+
+ trial_list = list()
+ for i in range(len(pkl_list)):
+ if len(pkl_list[i]) != 1:
+ raise ValueError("%s is more than !" % str(pkl_list))
+ fh = open(pkl_list[i][0])
+ trl = cPickle.load(fh)
+ fh.close()
+ trial_list.append(plot_util.extract_trials(trl))
+
+ sys.stdout.write("Plotting trace\n")
+ plot_optimization_trace(trial_list=trial_list, name_list=name_list, optimum=optimum,
+ title=title, log=log, save=save, y_max=y_max,
+ y_min=y_min, cut=cut)
+
+ if save != "":
+ sys.stdout.write("Saved plot to " + save + "\n")
+ else:
+ sys.stdout.write("..Done\n")
+
+if __name__ == "__main__":
+ prog = "python plotTrace.py WhatIsThis [WhatIsThis ]"
+ description = "Plot a Trace with evaluated points wrt to performance"
+
+ parser = ArgumentParser(description=description, prog=prog)
+
+ # Options for specific benchmarks
+ parser.add_argument("-o", "--optimum", type=float, dest="optimum",
+ default=0, help="If not set, the optimum is supposed to be zero")
+ # General Options
+ parser.add_argument("-l", "--log", action="store_true", dest="log",
+ default=False, help="Plot on log scale")
+ parser.add_argument("--max", dest="max", type=float,
+ default=0, help="Maximum of the plot")
+ parser.add_argument("--min", dest="min", type=float,
+ default=0, help="Minimum of the plot")
+ parser.add_argument("-s", "--save", dest="save",
+ default="", help="Where to save plot instead of showing it?")
+ parser.add_argument("-t", "--title", dest="title",
+ default="", help="Optional supertitle for plot")
+ parser.add_argument("-c", "--cut", default=sys.maxint, type=int,
+ help="Cut experiment pickles after a specified number of trials.")
+
+ args, unknown = parser.parse_known_args()
+
+ sys.stdout.write("Found " + str(len(unknown)) + " arguments\n")
+
+ pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown)
+
+ main(pkl_list=pkl_list_main, name_list=name_list_main, optimum=args.optimum,
+ title=args.title, log=args.log, save=args.save, y_max=args.max,
+ y_min=args.min, cut=args.cut)
diff --git a/HPOlib/Plotting/plotTraceWithStd_perEval.py b/HPOlib/Plotting/plotTraceWithStd_perEval.py
new file mode 100644
index 00000000..a7230061
--- /dev/null
+++ b/HPOlib/Plotting/plotTraceWithStd_perEval.py
@@ -0,0 +1,187 @@
+#!/usr/bin/env python
+
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+from argparse import ArgumentParser
+import cPickle
+import itertools
+import sys
+
+from matplotlib.pyplot import tight_layout, figure, subplots_adjust, subplot, savefig, show
+import matplotlib.gridspec
+import numpy as np
+
+from HPOlib.Plotting import plot_util
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+def plot_optimization_trace(trial_list, name_list, optimum=0, title="",
+ log=True, save="", y_max=0, y_min=0, scale_std=1):
+ markers =plot_util.get_plot_markers()
+ colors = plot_util.get_plot_colors()
+ linestyles = itertools.cycle(['-'])
+ size = 1
+
+ ratio = 5
+ gs = matplotlib.gridspec.GridSpec(ratio, 1)
+ fig = figure(1, dpi=100)
+ fig.suptitle(title, fontsize=16)
+ ax1 = subplot(gs[0:ratio, :])
+ ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
+ min_val = sys.maxint
+ max_val = -sys.maxint
+ max_trials = 0
+
+ trial_list_means = list()
+ trial_list_std = list()
+
+ # One trialList represents all runs from one optimizer
+ for i in range(len(trial_list)):
+ if log:
+ trial_list_means.append(np.log10(np.mean(np.array(trial_list[i]), axis=0)))
+ else:
+ trial_list_means.append(np.mean(np.array(trial_list[i]), axis=0))
+ trial_list_std.append(np.std(np.array(trial_list[i]), axis=0)*scale_std)
+
+ fig.suptitle(title, fontsize=16)
+
+ # Plot the average error and std
+ for i in range(len(trial_list_means)):
+ x = range(1, len(trial_list_means[i])+1)
+ y = trial_list_means[i] - optimum
+ m = markers.next()
+ c = colors.next()
+ l = linestyles.next()
+ std_up = y + trial_list_std[i]
+ std_down = y - trial_list_std[i]
+ ax1.fill_between(x, std_down, std_up,
+ facecolor=c, alpha=0.3, edgecolor=c)
+ ax1.plot(x, y, color=c, linewidth=size,
+ label=name_list[i][0] + "(" + str(len(trial_list[i])) + ")",
+ linestyle=l, marker=m)
+ if min(std_down) < min_val:
+ min_val = min(std_down)
+ if max(std_up) > max_val:
+ max_val = max(std_up)
+ if len(trial_list_means[i]) > max_trials:
+ max_trials = len(trial_list_means[i])
+
+ # Maybe plot on logscale
+ if scale_std != 1:
+ ylabel = ", %s * std" % scale_std
+ else:
+ ylabel = ""
+
+ if log:
+ ax1.set_ylabel("log10(Minfunction value)" + ylabel)
+ else:
+ ax1.set_ylabel("Minfunction value" + ylabel)
+
+ # Descript and label the stuff
+ leg = ax1.legend(loc='best', fancybox=True)
+ leg.get_frame().set_alpha(0.5)
+ ax1.set_xlabel("#Function evaluations")
+
+ if y_max == y_min:
+ # Set axes limits
+ ax1.set_ylim([min_val-0.1*abs((max_val-min_val)), max_val+0.1*abs((max_val-min_val))])
+ else:
+ ax1.set_ylim([y_min, y_max])
+ ax1.set_xlim([0, max_trials + 1])
+
+ tight_layout()
+ subplots_adjust(top=0.85)
+ if save != "":
+ savefig(save, dpi=100, facecolor='w', edgecolor='w',
+ orientation='portrait', papertype=None, format=None,
+ transparent=False, bbox_inches="tight", pad_inches=0.1)
+ else:
+ show()
+
+
+def main(pkl_list, name_list, autofill, optimum=0, save="", title="", log=False,
+ y_min=0, y_max=0, scale_std=1, cut=sys.maxint):
+
+ trial_list = list()
+ for i in range(len(pkl_list)):
+ trial_list.append(list())
+ for pkl in pkl_list[i]:
+ fh = open(pkl, "r")
+ trials = cPickle.load(fh)
+ fh.close()
+
+ trace = plot_util.extract_trajectory(trials, cut=cut)
+ trial_list[-1].append(np.array(trace))
+
+ for i in range(len(trial_list)):
+ max_len = max([len(ls) for ls in trial_list[i]])
+ for t in range(len(trial_list[i])):
+ if len(trial_list[i][t]) < max_len and autofill:
+ diff = max_len - len(trial_list[i][t])
+ # noinspection PyUnusedLocal
+ trial_list[i][t] = np.append(trial_list[i][t], [trial_list[i][t][-1] for x in range(diff)])
+ elif len(trial_list[i][t]) < max_len and not autofill:
+ raise ValueError("(%s != %s), Traces do not have the same length, please use -a" %
+ (str(max_len), str(len(trial_list[i][t]))))
+
+ plot_optimization_trace(trial_list, name_list, optimum, title=title, log=log,
+ save=save, y_min=y_min, y_max=y_max, scale_std=scale_std)
+
+ if save != "":
+ sys.stdout.write("Saved plot to " + save + "\n")
+ else:
+ sys.stdout.write("..Done\n")
+
+if __name__ == "__main__":
+ prog = "python plotTraceWithStd.py WhatIsThis [WhatIsThis ]"
+ description = "Plot a Trace with std for multiple experiments"
+
+ parser = ArgumentParser(description=description, prog=prog)
+
+ # Options for specific benchmarks
+ parser.add_argument("-o", "--optimum", type=float, dest="optimum",
+ default=0, help="If not set, the optimum is supposed to be zero")
+
+ # Options which are available only for this plot
+ parser.add_argument("-a", "--autofill", action="store_true", dest="autofill",
+ default=False, help="Fill trace automatically")
+ parser.add_argument("-c", "--scale", type=float, dest="scale",
+ default=1, help="Multiply std to get a nicer plot")
+ # General Options
+ parser.add_argument("-l", "--log", action="store_true", dest="log",
+ default=False, help="Plot on log scale")
+ parser.add_argument("--max", dest="max", type=float,
+ default=0, help="Maximum of the plot")
+ parser.add_argument("--min", dest="min", type=float,
+ default=0, help="Minimum of the plot")
+ parser.add_argument("-s", "--save", dest="save",
+ default="", help="Where to save plot instead of showing it?")
+ parser.add_argument("-t", "--title", dest="title",
+ default="", help="Optional supertitle for plot")
+
+ args, unknown = parser.parse_known_args()
+
+ sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n")
+
+ pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown)
+
+ main(pkl_list=pkl_list_main, name_list=name_list_main, autofill=args.autofill, optimum=args.optimum,
+ save=args.save, title=args.title, log=args.log, y_min=args.min, y_max=args.max, scale_std=args.scale)
diff --git a/HPOlib/Plotting/plotTraceWithStd_perTime.py b/HPOlib/Plotting/plotTraceWithStd_perTime.py
new file mode 100644
index 00000000..e02249b6
--- /dev/null
+++ b/HPOlib/Plotting/plotTraceWithStd_perTime.py
@@ -0,0 +1,256 @@
+#!/usr/bin/env python
+
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+from argparse import ArgumentParser
+import cPickle
+import itertools
+import sys
+
+from matplotlib.pyplot import tight_layout, figure, subplots_adjust, subplot, savefig, show
+import matplotlib.gridspec
+import numpy as np
+
+from HPOlib.Plotting import plot_util
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+def plot_optimization_trace(trial_list, name_list, times_list, optimum=0, title="",
+ log=True, save="", y_max=0, y_min=0, scale_std=1):
+ markers = plot_util.get_plot_markers()
+ colors = plot_util.get_plot_colors()
+ linestyles = itertools.cycle(['-'])
+ size = 1
+
+ ratio = 5
+ gs = matplotlib.gridspec.GridSpec(ratio, 1)
+ fig = figure(1, dpi=100)
+ fig.suptitle(title, fontsize=16)
+ ax1 = subplot(gs[0:ratio, :])
+ ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
+ min_val = sys.maxint
+ max_val = -sys.maxint
+ max_trials = 0
+
+ trial_list_means = list()
+ trial_list_std = list()
+
+ # One trialList represents all runs from one optimizer
+ for i in range(len(trial_list)):
+ if log:
+ trial_list_means.append(np.log10(np.mean(np.array(trial_list[i]), axis=0)))
+ else:
+ trial_list_means.append(np.mean(np.array(trial_list[i]), axis=0))
+ trial_list_std.append(np.std(np.array(trial_list[i]), axis=0)*scale_std)
+ times_list[i] = np.array(times_list[i])
+
+ fig.suptitle(title, fontsize=16)
+
+ # Plot the average error and std
+ for i in range(len(trial_list_means)):
+ x = times_list[i]
+ y = trial_list_means[i] - optimum
+ # m = markers.next()
+ c = colors.next()
+ l = linestyles.next()
+ std_up = y + trial_list_std[i]
+ std_down = y - trial_list_std[i]
+
+ ax1.fill_between(x, std_down, std_up,
+ facecolor=c, alpha=0.3, edgecolor=c)
+ ax1.plot(x, y, color=c, linewidth=size*2,
+ label=name_list[i][0] + "(" + str(len(trial_list[i])) + ")",
+ linestyle=l, marker="")
+ if min(std_down) < min_val:
+ min_val = min(std_down)
+ if max(y + std_up) > max_val:
+ max_val = max(std_up)
+ if max(times_list[i]) > max_trials:
+ max_trials = max(times_list[i])
+
+ # Maybe plot on logscale
+ if scale_std != 1:
+ ylabel = ", %s * std" % scale_std
+ else:
+ ylabel = ""
+
+ if log:
+ ax1.set_ylabel("log10(Minfunction value)" + ylabel)
+ else:
+ ax1.set_ylabel("Minfunction value" + ylabel)
+
+ # Descript and label the stuff
+ leg = ax1.legend(loc='best', fancybox=True)
+ leg.get_frame().set_alpha(0.5)
+ ax1.set_xlabel("Duration [sec] ")
+
+ if y_max == y_min:
+ # Set axes limit
+ ax1.set_ylim([min_val-0.1*abs((max_val-min_val)), max_val+0.1*abs((max_val-min_val))])
+ else:
+ ax1.set_ylim([y_min, y_max])
+ ax1.set_xlim([0, max_trials])
+
+ tight_layout()
+ subplots_adjust(top=0.85)
+ if save != "":
+ savefig(save, dpi=100, facecolor='w', edgecolor='w',
+ orientation='portrait', papertype=None, format=None,
+ transparent=False, bbox_inches="tight", pad_inches=0.1)
+ else:
+ show()
+
+
+def fill_trajectories(trace_list, times_list):
+ """ Each trajectory need to has the exact same number of entries and timestamps"""
+ # We need to define the max value = what is measured before the first evaluation
+ max_value = np.max([np.max(ls) for ls in trace_list])
+
+ number_exp = len(trace_list)
+ new_trajectories = list()
+ new_times = list()
+ for i in range(number_exp):
+ new_trajectories.append(list())
+ new_times.append(list())
+ # noinspection PyUnusedLocal
+ counter = [1 for i in range(number_exp)]
+ finish = False
+
+ # We need to insert the max values in the beginning and the min values in the end
+ for i in range(number_exp):
+ trace_list[i].insert(0, max_value)
+ trace_list[i].append(np.min(trace_list[i]))
+ times_list[i].insert(0, 0)
+ times_list[i].append(sys.maxint)
+
+ # Add all possible time values
+ while not finish:
+ min_idx = np.argmin([times_list[idx][counter[idx]] for idx in range(number_exp)])
+ counter[min_idx] += 1
+ for idx in range(number_exp):
+ new_times[idx].append(times_list[min_idx][counter[min_idx] - 1])
+ new_trajectories[idx].append(trace_list[idx][counter[idx] - 1])
+ # Check if we're finished
+ for i in range(number_exp):
+ finish = True
+ if counter[i] < len(trace_list[i]) - 1:
+ finish = False
+ break
+
+ times = new_times
+ trajectories = new_trajectories
+ tmp_times = list()
+
+ # Sanitize lists and delete double entries
+ for i in range(number_exp):
+ tmp_times = list()
+ tmp_traj = list()
+ for t in range(len(times[i]) - 1):
+ if times[i][t+1] != times[i][t] and not np.isnan(times[i][t]):
+ tmp_times.append(times[i][t])
+ tmp_traj.append(trajectories[i][t])
+ tmp_times.append(times[i][-1])
+ tmp_traj.append(trajectories[i][-1])
+ times[i] = tmp_times
+ trajectories[i] = tmp_traj
+
+ # We need only one list for all times
+ times = tmp_times
+
+ return trajectories, times
+
+
+def main(pkl_list, name_list, autofill, optimum=0, save="", title="",
+ log=False, y_min=0, y_max=0, scale_std=1, cut=sys.maxint):
+
+ trial_list = list()
+ times_list = list()
+
+ for i in range(len(pkl_list)):
+ tmp_trial_list = list()
+ tmp_times_list = list()
+ for pkl in pkl_list[i]:
+ fh = open(pkl, "r")
+ trials = cPickle.load(fh)
+ fh.close()
+
+ trace = plot_util.extract_trajectory(trials)
+ times = plot_util.extract_runtime_timestamps(trials)
+ tmp_times_list.append(times)
+ tmp_trial_list.append(trace)
+ # We feed this function with two lists of lists and get one list of lists and one list
+ tmp_trial_list, tmp_times_list = fill_trajectories(tmp_trial_list, tmp_times_list)
+ trial_list.append(tmp_trial_list)
+ times_list.append(tmp_times_list)
+
+ for i in range(len(trial_list)):
+ max_len = max([len(ls) for ls in trial_list[i]])
+ for t in range(len(trial_list[i])):
+ if len(trial_list[i][t]) < max_len and autofill:
+ diff = max_len - len(trial_list[i][t])
+ # noinspection PyUnusedLocal
+ trial_list[i][t] = np.append(trial_list[i][t], [trial_list[i][t][-1] for x in range(diff)])
+ elif len(trial_list[i][t]) < max_len and not autofill:
+ raise ValueError("(%s != %s), Traces do not have the same length, please use -a" %
+ (str(max_len), str(len(trial_list[i][t]))))
+
+ plot_optimization_trace(trial_list, name_list, times_list, optimum, title=title, log=log,
+ save=save, y_min=y_min, y_max=y_max, scale_std=scale_std)
+ if save != "":
+ sys.stdout.write("Saved plot to " + save + "\n")
+ else:
+ sys.stdout.write("..Done\n")
+
+if __name__ == "__main__":
+ prog = "python plotTraceWithStd.py WhatIsThis [WhatIsThis ]"
+ description = "Plot a Trace with std for multiple experiments"
+
+ parser = ArgumentParser(description=description, prog=prog)
+
+ # Options for specific benchmarks
+ parser.add_argument("-o", "--optimum", type=float, dest="optimum",
+ default=0, help="If not set, the optimum is supposed to be zero")
+
+ # Options which are available only for this plot
+ parser.add_argument("-a", "--autofill", action="store_true", dest="autofill",
+ default=False, help="Fill trace automatically")
+ parser.add_argument("-c", "--scale", type=float, dest="scale",
+ default=1, help="Multiply std to get a nicer plot")
+ # General Options
+ parser.add_argument("-l", "--log", action="store_true", dest="log",
+ default=False, help="Plot on log scale")
+ parser.add_argument("--max", dest="max", type=float,
+ default=0, help="Maximum of the plot")
+ parser.add_argument("--min", dest="min", type=float,
+ default=0, help="Minimum of the plot")
+ parser.add_argument("-s", "--save", dest="save",
+ default="", help="Where to save plot instead of showing it?")
+ parser.add_argument("-t", "--title", dest="title",
+ default="", help="Optional supertitle for plot")
+
+ args, unknown = parser.parse_known_args()
+
+ sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n")
+
+ pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown)
+
+ main(pkl_list_main, name_list_main, autofill=args.autofill, optimum=args.optimum, save=args.save,
+ title=args.title, log=args.log, y_min=args.min, y_max=args.max, scale_std=args.scale)
diff --git a/HPOlib/Plotting/plot_util.py b/HPOlib/Plotting/plot_util.py
new file mode 100644
index 00000000..4f148bd4
--- /dev/null
+++ b/HPOlib/Plotting/plot_util.py
@@ -0,0 +1,136 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import cPickle
+import itertools
+import os
+import numpy as np
+import sys
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+def get_plot_markers():
+ return itertools.cycle(['o', 's', 'x', '^'])
+
+
+def get_plot_colors():
+ # color brewer, 2nd qualitative 9 color scheme (http://colorbrewer2.org/)
+ return itertools.cycle(["#e41a1c", # Red
+ "#377eb8", # Blue
+ "#4daf4a", # Green
+ "#984ea3", # Purple
+ "#ff7f00", # Orange
+ "#ffff33", # Yellow
+ "#a65628", # Brown
+ "#f781bf", # Pink
+ "#999999"]) # Grey
+
+
+def read_pickles(name_list, pkl_list, cut=sys.maxint):
+ best_dict = dict()
+ idx_dict = dict()
+ keys = list()
+ for i in range(len(name_list)):
+ keys.append(name_list[i][0])
+ best_dict[name_list[i][0]] = list()
+ idx_dict[name_list[i][0]] = list()
+ for pkl in pkl_list[i]:
+ fh = open(pkl)
+ trial = cPickle.load(fh)
+ fh.close()
+ best, idx = get_best_value_and_index(trial, cut)
+ best_dict[name_list[i][0]].append(best)
+ idx_dict[name_list[i][0]].append(idx)
+ return best_dict, idx_dict, keys
+
+
+def get_pkl_and_name_list(argument_list):
+ name_list = list()
+ pkl_list = list()
+ now_data = False
+ for i in range(len(argument_list)):
+ if not ".pkl" in argument_list[i] and now_data:
+ raise ValueError("You need at least on .pkl file per Experiment, %s has none" % name_list[-1])
+ elif not ".pkl" in argument_list[i] and not now_data:
+ print "Adding", argument_list[i]
+ name_list.append([argument_list[i], 0])
+ pkl_list.append(list())
+ now_data = True
+ continue
+ else:
+ if os.path.exists(argument_list[i]):
+ now_data = False
+ name_list[-1][1] += 1
+ pkl_list[-1].append(argument_list[i])
+ else:
+ raise ValueError("%s is not a valid file" % argument_list[i])
+ if now_data:
+ raise ValueError("You need at least one .pkl file per Experiment, %s has none" % name_list[-1])
+ return pkl_list, name_list
+
+
+def extract_trajectory(trials, cut=sys.maxint):
+ trace = list()
+ currentbest = trials['trials'][0]
+
+ for result in [trial["result"] for trial in trials['trials'][:cut]]:
+ if result < currentbest:
+ currentbest = result
+ trace.append(currentbest)
+ return trace
+
+
+def extract_trials(trials, cut=sys.maxint):
+ trl = [trial["result"] for trial in trials['trials'][:cut]]
+ return trl
+
+
+def extract_runtime_timestamps(trials, cut=sys.maxint):
+ # return a list like (20, 53, 101, 200)
+ time_list = list()
+ time_list.append(0)
+ for trial in trials["trials"][:cut]:
+ time_list.append(np.sum(trial["instance_durations"]) + time_list[-1])
+ return time_list
+
+
+def get_best(trials, cut=False):
+ # returns the best value found in this experiment
+ traj = extract_trajectory(trials)
+ best_value = sys.maxint
+ if cut and 0 < cut < len(traj):
+ best_value = traj[cut]
+ else:
+ best_value = traj[-1]
+ return best_value
+
+
+def get_best_value_and_index(trials, cut=False):
+ # returns the best value and corresponding index
+ traj = extract_trajectory(trials)
+ best_value = sys.maxint
+ best_index = -1
+ if cut and 0 < cut < len(traj):
+ best_value = traj[cut]
+ best_index = np.argmin(traj[:cut])
+ else:
+ best_value = traj[-1]
+ best_index = np.argmin(traj)
+ return best_value, best_index
\ No newline at end of file
diff --git a/HPOlib/Plotting/results.sh b/HPOlib/Plotting/results.sh
new file mode 100644
index 00000000..fea13d2a
--- /dev/null
+++ b/HPOlib/Plotting/results.sh
@@ -0,0 +1,163 @@
+
+
+directory=`ls | grep "^smac_2_06_01-dev\>"`
+if [ -a "${directory}" ]
+ then
+ echo "Looking for SMAC (smac_2_06_01-dev):"
+
+ printf "%5s | %4s(%5s) | %5s | %10s | %10s\n" "Seed" "#run" "crash" "#iter" "Performance" "Test-Perf"
+ for i in `seq 1000 1000 10000`
+ do
+ directory=`ls | grep "^smac_2_06_01-dev_${i}_"`
+ if [ -f "${directory}/smac_2_06_01-dev.out" ]
+ then
+ it=`cat ${directory}/smac*.out | grep "Model/Iteration used:" | tail -1`
+ it=`echo $it | cut -d' ' -f3`
+
+ per=`cat ${directory}/smac*.out | grep "Performance of the Incumbent:" | tail -1`
+ per=`echo $per | cut -d' ' -f5`
+
+ num=`cat ${directory}/smac*.out | grep "Algorithm Runs used:" | tail -1`
+ num=`echo $num | cut -d' ' -f4`
+
+ numC=`ls ${directory}/ | grep 'instance.out$' | wc -l`
+ numC=$(($numC - 1))
+
+ test_error=999
+ test_file=`ls "$directory/" | grep "_test_run.out"`
+ if [ -f "${directory}/$test_file" ]
+ then
+ test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1`
+ test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1`
+ fi
+ printf "%5s | %4s(%5s) | %5s | %10f | %10f\n" "$i" "$num" "$numC" "$it" "$per" "$test_error"
+ fi
+
+ done
+fi
+
+
+directory=`ls | grep "^partial_smac\>"`
+if [ -a "${directory}" ]
+then
+
+ echo "Looking for SMAC (smac-v2_06_02-partial38):"
+
+ printf "%5s | %4s(%5s) | %5s | %10s | %10s\n" "Seed" "#run" "crash" "#iter" "Performance" "Test-Perf"
+ for i in `seq 1000 1000 10000`
+ do
+ directory=`ls | grep "^partial_smac_${i}_"`
+ if [ -f "${directory}/partial_smac.out" ]
+ then
+ it=`cat ${directory}/partial_*.out | grep "Model/Iteration used:" | tail -1`
+ it=`echo $it | cut -d' ' -f3`
+
+ per=`cat ${directory}/partial_*.out | grep "Performance of the Incumbent:" | tail -1`
+ per=`echo $per | cut -d' ' -f5`
+
+ num=`cat ${directory}/partial_*.out | grep "Algorithm Runs used:" | tail -1`
+ num=`echo $num | cut -d' ' -f4`
+
+ numC=`ls ${directory}/ | grep 'instance.out$' | wc -l`
+ numC=$(($numC - 1))
+
+ test_error=999
+ test_file=`ls "$directory/" | grep "_test_run.out"`
+ if [ -f "${directory}/$test_file" ]
+ then
+ test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1`
+ test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1`
+ fi
+ printf "%5s | %4s(%5s) | %5s | %10f | %10f\n" "$i" "$num" "$numC" "$it" "$per" "$test_error"
+ fi
+
+ done
+fi
+
+
+directory=`ls | grep "^hyperopt_august2013_mod\>"`
+if [ -a "${directory}" ]
+then
+ echo "Looking for TPE:"
+
+ printf "%5s | %4s(%5s) | %10s | %10s\n" "Seed" "#run" "crash" "Performance" "Test-Perf"
+ for i in `seq 1000 1000 10000`
+ do
+ directory=`ls | grep "^hyperopt_august2013_mod_${i}_"`
+ if [ -a "${directory}/hyperopt_august2013_mod.out" ]
+ then
+ num=`cat ${directory}/hyperopt_august2013_mod.out | grep " -----------------------RUNNING RUNSOLVER" | wc -l`
+
+ per=`cat ${directory}/hyperopt_august2013_mod.out | grep "Result for ParamILS:" | sort -r | tail -1`
+ per=`echo $per | cut -d' ' -f9`
+ per=`echo $per | sed 's/,//'`
+
+ numC=`ls ${directory}/ | grep 'instance.out$' | wc -l`
+ numC=$(($numC - 1))
+
+ test_error=999
+ test_file=`ls -1 "$directory" | grep "_test_run.out"`
+ printf "%5s | %4s(%5s) | %10f \n" "$i" "$num" "$numC" "$per"
+ fi
+ done
+fi
+
+
+directory=`ls | grep "^random_hyperopt_august2013_mod\>"`
+if [ -a "${directory}" ]
+then
+ echo "Looking for RandomTPE:"
+
+ printf "%5s | %4s(%5s) | %10s | %10s\n" "Seed" "#run" "crash" "Performance" "Test-Perf"
+ for i in `seq 1000 1000 10000`
+ do
+ directory=`ls | grep "^random_hyperopt_august2013_mod_${i}_"`
+ if [ -a "${directory}/random_hyperopt_august2013_mod.out" ]
+ then
+ num=`cat ${directory}/random_hyperopt_august2013_mod.out | grep " -----------------------RUNNING RUNSOLVER" | wc -l`
+
+ per=`cat ${directory}/random_hyperopt_august2013_mod.out | grep "Result for ParamILS:" | sort -r | tail -1`
+ per=`echo $per | cut -d' ' -f9`
+ per=`echo $per | sed 's/,//'`
+
+ numC=`ls ${directory}/ | grep 'instance.out$' | wc -l`
+ numC=$(($numC - 1))
+
+ test_error=999
+ test_file=`ls -1 "$directory" | grep "_test_run.out"`
+ printf "%5s | %4s(%5s) | %10f \n" "$i" "$num" "$numC" "$per"
+ fi
+ done
+fi
+
+
+directory=`ls | grep "^spearmint_april2013_mod\>"`
+if [ -a "${directory}" ]
+ then
+ echo "Looking for Spearmint:"
+
+ printf "%5s | %4s(%5s) | %10s | %10s\n" "Seed" "#run" "crash" "Performance" "Test-Perf"
+ for i in `seq 1000 1000 10000`
+ do
+ directory=`ls | grep "^spearmint_april2013_mod_${i}_"`
+ if [ -a "${directory}/spearmint_april2013_mod.out" ]
+ then
+ num=`cat ${directory}/spearmint_april2013_mod.out | grep " pending " | tail -1`
+ num=`echo $num | cut -d' ' -f5`
+ per=`cat ${directory}/spearmint_april2013_mod.out | grep "best:" | tail -1`
+ per=`echo $per | cut -d' ' -f3`
+
+ numC=`ls ${directory}/ | grep 'instance.out$' | wc -l`
+ numC=$(($numC - 1))
+
+ test_error=999
+ test_file=`ls -1 "$directory" | grep "_test_run.out"`
+ if [ -f "$directory/$test_file" ]
+ then
+ test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1`
+ test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1`
+ fi
+ printf "%5s | %4s(%5s) | %10f | %10f\n" "$i" "$num" "$numC" "$per" "$test_error"
+ fi
+ done
+fi
diff --git a/HPOlib/Plotting/statistics.py b/HPOlib/Plotting/statistics.py
new file mode 100644
index 00000000..62b14100
--- /dev/null
+++ b/HPOlib/Plotting/statistics.py
@@ -0,0 +1,178 @@
+#!/usr/bin/env python
+
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+from argparse import ArgumentParser
+import cPickle
+import sys
+
+import numpy
+import scipy
+from scipy import stats
+
+from HPOlib.Plotting import plot_util
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+def _get_best_trial(filename, cut=None):
+ try:
+ fh = open(filename, "r")
+ trials = cPickle.load(fh)
+ fh.close()
+
+ current_best = numpy.Inf
+ best_idx = 0
+ if cut is None:
+ cut = len(trials['trials'])
+ print filename, "#Trials", len(trials['trials'])
+ for i, trial in enumerate(trials['trials'][:cut]):
+ result = trial['result']
+ if result < current_best:
+ best_idx = i
+ current_best = result
+ if current_best == numpy.Inf:
+ raise Exception("%s does not contain any results" % filename)
+ return current_best, best_idx
+ except Exception as e:
+ print "Problem with ", filename, e
+ sys.stdout.flush()
+ return None, None
+
+# TODO: Don't know whether this is longer needed
+"""
+def _mann_whitney_u(x, y=None):
+ # Calculate the Mann-Whitney-U test.
+ # The Wilcoxon signed-rank test tests the null hypothesis that two related paired
+ # samples come from the same distribution. In particular, it tests whether the
+ # distribution of the differences x - y is symmetric about zero.
+ # It is a non-parametric version of the paired T-test.
+
+ # A significance-dict for a two-tailed test with 0.05 confidence
+ # from http://de.wikipedia.org/wiki/Wilcoxon-Mann-Whitney-Test
+ significance_table = \
+ [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2],
+ [0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8],
+ [0, 0, 0, 0, 1, 2, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 11, 12, 13, 14],
+ [0, 0, 0, 0, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13, 14, 15, 17, 18, 19, 20],
+ [0, 0, 0, 0, 0, 5, 6, 8, 10, 11, 13, 14, 16, 17, 19, 21, 22, 24, 25, 27],
+ [0, 0, 0, 0, 0, 0, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34],
+ [0, 0, 0, 0, 0, 0, 0, 13, 15, 17, 19, 22, 24, 26, 29, 31, 34, 36, 38, 41],
+ [0, 0, 0, 0, 0, 0, 0, 0, 17, 20, 23, 26, 28, 31, 34, 37, 39, 42, 45, 48],
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 26, 29, 33, 36, 39, 42, 45, 48, 52, 55],
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 33, 37, 40, 44, 47, 51, 55, 58, 62],
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 41, 45, 49, 53, 57, 61, 65, 69],
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 50, 54, 59, 63, 67, 72, 76],
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 59, 64, 69, 74, 78, 83],
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 70, 75, 80, 85, 90],
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 81, 86, 92, 98],
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 93, 99, 105],
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 106, 112],
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 119],
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127]]
+
+ u_value, p = scipy.stats.mannwhitneyu(x, y)
+
+ # noinspection PyNoneFunctionAssignment
+ x = asarray(x).compressed().view(numpy.ndarray)
+ y = asarray(y).compressed().view(numpy.ndarray)
+
+ n1 = len(x)
+ n2 = len(y)
+ print n1, n2
+ if n1 > n2:
+ tmp = n2
+ n2 = n1
+ n1 = tmp
+
+ if n1 < 5 or n2 < 5:
+ return 10000, 10000
+ if n1 < 20 or n2 < 20:
+ print "WARNING: scipy.stat might not be accurate, p value is %f and significance according to table is %s" % \
+ (p, u_value <= significance_table[n1 - 1][n2 - 1])
+ return u_value, p
+"""
+
+
+def main(pkl_list, name_list, cut=sys.maxint):
+ best_dict, idx_dict, keys = plot_util.read_pickles(name_list, pkl_list,
+ cut=cut)
+
+ for k in keys:
+ sys.stdout.write("%10s: %s experiment(s)\n" % (k, len(best_dict[k])))
+
+ sys.stdout.write("Unpaired t-tests-----------------------------------------------------\n")
+ # TODO: replace by itertools
+ for idx, k in enumerate(keys):
+ if len(keys) > 1:
+ for j in keys[idx+1:]:
+ t_true, p_true = stats.ttest_ind(best_dict[k], best_dict[j])
+ rounded_t_true, rounded_p_true = stats.ttest_ind(numpy.round(best_dict[k], 3),
+ numpy.round(best_dict[j], 3))
+
+ sys.stdout.write("%10s vs %10s\n" % (k, j))
+ sys.stdout.write("Standard independent 2 sample test, equal population variance\n")
+ sys.stdout.write(" "*24 + " T: %10.5e, p-value: %10.5e (%5.3f%%) \n" %
+ (t_true, p_true, p_true*100))
+ sys.stdout.write("Rounded: ")
+ sys.stdout.write(" T: %10.5e, p-value: %10.5e (%5.3f%%)\n" %
+ (rounded_t_true, rounded_p_true, rounded_p_true*100))
+ if tuple(map(int, (scipy.__version__.split(".")))) >= (0, 11, 0):
+ # print scipy.__version__ >= '0.11.0'
+ t_false, p_false = stats.ttest_ind(best_dict[k], best_dict[j], equal_var=False)
+ rounded_t_false, rounded_p_false = stats.ttest_ind(numpy.round(best_dict[k], 3),
+ numpy.round(best_dict[j], 3),
+ equal_var=False)
+ sys.stdout.write("Welch's t-test, no equal population variance\n")
+ sys.stdout.write(" "*24)
+ sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" %
+ (t_false, p_false, p_false*100))
+ sys.stdout.write("Rounded: ")
+ sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" %
+ (rounded_t_false, rounded_p_false, rounded_p_false*100))
+ sys.stdout.write("\n")
+
+ sys.stdout.write("Best Value-----------------------------------------------------------\n")
+ for k in keys:
+ sys.stdout.write("%10s: %10.5f (min: %10.5f, max: %10.5f, std: %5.3f)\n" %
+ (k, float(numpy.mean(best_dict[k])), float(numpy.min(best_dict[k])),
+ numpy.max(best_dict[k]), float(numpy.std(best_dict[k]))))
+
+ sys.stdout.write("Needed Trials--------------------------------------------------------\n")
+ for k in keys:
+ sys.stdout.write("%10s: %10.5f (min: %10.5f, max: %10.5f, std: %5.3f)\n" %
+ (k, float(numpy.mean(idx_dict[k])), float(numpy.min(idx_dict[k])),
+ numpy.max(idx_dict[k]), float(numpy.std(idx_dict[k]))))
+
+ sys.stdout.write("------------------------------------------------------------------------\n")
+
+if __name__ == "__main__":
+ prog = "python statistics.py WhatIsThis WhatIsThis [WhatIsThis ]"
+ description = "Return some statistical information"
+
+ parser = ArgumentParser(description=description, prog=prog)
+
+ parser.add_argument("-c", "--cut", dest="cut", default=sys.maxint,
+ type=int, help="Only consider that many evaluations")
+ args, unknown = parser.parse_known_args()
+
+ pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown)
+ main(pkl_list=pkl_list_main, name_list=name_list_main, cut=args.cut)
diff --git a/HPOlib/__init__.py b/HPOlib/__init__.py
new file mode 100644
index 00000000..64ca9bc7
--- /dev/null
+++ b/HPOlib/__init__.py
@@ -0,0 +1,4 @@
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+__version__ = "0.1.0rc1"
diff --git a/HPOlib/benchmark_functions.py b/HPOlib/benchmark_functions.py
new file mode 100644
index 00000000..5a2eb531
--- /dev/null
+++ b/HPOlib/benchmark_functions.py
@@ -0,0 +1,1915 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import numpy as np
+import sys
+
+
+def save_branin(params, **kwargs):
+ if "x" not in params or "y" not in params:
+ raise ValueError("No params found ['x', 'y']\n")
+ x = float(params["x"])
+ y = float(params["y"])
+ if type(x) == np.ndarray or type(y) == np.ndarray:
+ x = x[0]
+ y = y[0]
+
+ if -5 > x or x > 10:
+ raise ValueError("X value not in between -5 and 10")
+ if 0 > y or y > 15:
+ raise ValueError("Y value not in between 0 and 15")
+ return branin(x, y)
+
+
+def branin(x, y):
+ """Branin test function
+
+ The number of variables n = 2.
+ constraints:
+ -5 <= x <= 10, 0 <= y <= 15
+ three global optima: (-pi, 12.275), (pi, 2.275), (9.42478, 2.475), where
+ branin = 0.397887"""
+
+ result = (y-(5.1/(4*np.pi**2))*x**2+5*x/np.pi-6)**2
+ result += 10*(1-1/(8*np.pi))*np.cos(x)+10
+ return result
+
+
+def save_har6(params, **kwargs):
+ if "x" not in params or "y" not in params or "z" not in params \
+ or "a" not in params or "b" not in params or "c" not in params:
+ sys.stderr.write("No params found ['x', 'y']\n")
+ return np.NaN
+ x = float(params["x"])
+ y = float(params["y"])
+ z = float(params["z"])
+ a = float(params["a"])
+ b = float(params["b"])
+ c = float(params["c"])
+ if type(x) == np.ndarray:
+ x = x[0]
+ y = y[0]
+ z = z[0]
+ a = a[0]
+ b = b[0]
+ c = c[0]
+ return har6(x, y, z, a, b, c)
+
+
+def har6(x, y, z, a, b, c):
+ """6d Hartmann test function
+
+ constraints:
+ 0 <= xi <= 1, i = 1..6
+ global optimum at (0.20169, 0.150011, 0.476874, 0.275332, 0.311652, 0.6573),
+ where har6 = -3.32236"""
+
+ value = np.array([x, y, z, a, b, c])
+
+ if 0 > x or x > 1 or 0 > y or y > 1 or 0 > z or z > 1:
+ raise ValueError("x=%s, y=%s or z=%s not in between 0 and 1" %
+ (x, y, z))
+ if 0 > a or a > 1 or 0 > b or b > 1 or 0 > c or c > 1:
+ raise ValueError("a=%s, b=%s or c=%s not in between 0 and 1" %
+ (a, b, c))
+
+ a = np.array([[10.0, 3.0, 17.0, 3.5, 1.7, 8.0],
+ [ 0.05, 10.0, 17.0, 0.1, 8.0, 14.0],
+ [ 3.0, 3.5, 1.7, 10.0, 17.0, 8.0],
+ [17.0, 8.0, 0.05, 10.0, 0.1, 14.0]])
+ c = np.array([1.0, 1.2, 3.0, 3.2])
+ p = np.array([[0.1312, 0.1696, 0.5569, 0.0124, 0.8283, 0.5886],
+ [0.2329, 0.4135, 0.8307, 0.3736, 0.1004, 0.9991],
+ [0.2348, 0.1451, 0.3522, 0.2883, 0.3047, 0.6650],
+ [0.4047, 0.8828, 0.8732, 0.5743, 0.1091, 0.0381]])
+ s = 0
+ for i in [0,1,2,3]:
+ sm = a[i,0]*(value[0]-p[i,0])**2
+ sm += a[i,1]*(value[1]-p[i,1])**2
+ sm += a[i,2]*(value[2]-p[i,2])**2
+ sm += a[i,3]*(value[3]-p[i,3])**2
+ sm += a[i,4]*(value[4]-p[i,4])**2
+ sm += a[i,5]*(value[5]-p[i,5])**2
+ s += c[i]*np.exp(-sm)
+ result = -s
+ return result
+
+
+def save_lda_on_grid(params, ret_time=False, **kwargs):
+ if "Kappa" not in params or "Tau" not in params or "S" not in params:
+ sys.stderr.write("No params found ['Kappa', 'Tau', 'S']: %s\n" % str(params))
+ return np.NaN
+ kappa = int(float(params["Kappa"]))
+ tau = int(float(params["Tau"]))
+ s = int(float(params["S"]))
+
+ if type(kappa) == np.ndarray:
+ kappa = kappa[0]
+ tau = tau[0]
+ s = s[0]
+ return lda_on_grid(kappa, tau, s, ret_time=ret_time)
+
+
+def lda_on_grid(kappa, tau, s, ret_time=False):
+ # Values for an 6*8*8 grid search which was performed by Hofman et. al.
+ # Values obtained from Jasper Snoek
+ # Kappa Tau S Value Time
+ # ret_time: return time instead of perplexity
+ configurations = np.array([[1.000000, 4.000000, 16.000000, 2014.255351, 36393.190000],
+ [0.900000, 1024.000000, 4096.000000, 1680.540179, 36419.510000],
+ [0.600000, 1024.000000, 4096.000000, 1328.191297, 24219.850000],
+ [0.600000, 16.000000, 4.000000, 2859.440420, 18862.010000],
+ [0.700000, 1.000000, 4096.000000, 1313.263743, 11218.420000],
+ [0.500000, 64.000000, 16384.000000, 1271.869567, 18228.940000],
+ [0.600000, 64.000000, 4.000000, 2630.424468, 18970.510000],
+ [1.000000, 1.000000, 1.000000, 2547.784743, 36317.510000],
+ [1.000000, 16.000000, 64.000000, 1558.076377, 29985.160000],
+ [0.500000, 256.000000, 16384.000000, 1278.123799, 20822.350000],
+ [0.800000, 256.000000, 4096.000000, 1361.320422, 30758.540000],
+ [0.600000, 256.000000, 16.000000, 1800.429160, 10775.890000],
+ [0.500000, 4.000000, 64.000000, 2084.897678, 14864.830000],
+ [0.800000, 64.000000, 256.000000, 1349.812010, 17315.100000],
+ [1.000000, 16.000000, 4.000000, 2079.802409, 36338.350000],
+ [0.500000, 1.000000, 4.000000, 3624.167804, 12595.790000],
+ [0.900000, 16.000000, 16.000000, 1749.505296, 27772.590000],
+ [0.800000, 1.000000, 4096.000000, 1310.060860, 15432.120000],
+ [1.000000, 64.000000, 1024.000000, 1343.801912, 19662.620000],
+ [0.700000, 1.000000, 64.000000, 1983.587762, 10802.270000],
+ [0.500000, 4.000000, 16.000000, 2804.748620, 12678.520000],
+ [0.700000, 256.000000, 64.000000, 1427.856381, 10884.970000],
+ [0.900000, 16.000000, 1024.000000, 1317.263697, 15282.860000],
+ [0.500000, 256.000000, 1.000000, 5000.332116, 12602.320000],
+ [1.000000, 1024.000000, 4.000000, 1637.661866, 36225.830000],
+ [0.800000, 1024.000000, 4096.000000, 1537.873629, 36424.780000],
+ [0.900000, 16.000000, 4.000000, 2040.531678, 36327.940000],
+ [0.500000, 64.000000, 1.000000, 5022.511473, 14693.930000],
+ [1.000000, 4.000000, 256.000000, 1472.690781, 25890.240000],
+ [1.000000, 64.000000, 16.000000, 1572.944521, 36307.320000],
+ [0.800000, 1024.000000, 4.000000, 1528.555961, 19240.020000],
+ [0.800000, 64.000000, 16.000000, 1631.072354, 15148.720000],
+ [0.900000, 64.000000, 16.000000, 1541.968527, 25795.950000],
+ [0.900000, 1024.000000, 256.000000, 1468.342464, 25968.680000],
+ [0.700000, 4.000000, 1.000000, 2968.382913, 27098.820000],
+ [0.900000, 1.000000, 64.000000, 1886.883404, 23495.740000],
+ [0.800000, 4.000000, 16.000000, 2101.979941, 14956.600000],
+ [0.600000, 1.000000, 16.000000, 2601.345619, 10651.960000],
+ [0.900000, 1024.000000, 64.000000, 1506.415675, 21523.900000],
+ [0.700000, 16.000000, 16384.000000, 1271.574590, 24894.000000],
+ [1.000000, 1.000000, 4.000000, 2524.250165, 36327.700000],
+ [1.000000, 1024.000000, 16384.000000, 2250.711024, 36354.110000],
+ [0.800000, 4.000000, 1.000000, 2566.382537, 29395.750000],
+ [0.600000, 64.000000, 64.000000, 1630.248431, 8734.110000],
+ [0.800000, 16.000000, 16.000000, 1836.733098, 17066.550000],
+ [0.500000, 16.000000, 16384.000000, 1266.167382, 16119.520000],
+ [0.500000, 1024.000000, 256.000000, 1364.836274, 10978.410000],
+ [0.600000, 1.000000, 1024.000000, 1435.221761, 8879.080000],
+ [0.600000, 4.000000, 4.000000, 3053.489799, 18842.800000],
+ [0.600000, 64.000000, 16384.000000, 1268.070557, 22757.820000],
+ [1.000000, 4.000000, 16384.000000, 1293.198126, 27337.720000],
+ [1.000000, 4.000000, 64.000000, 1718.817531, 30082.460000],
+ [0.600000, 1024.000000, 4.000000, 2045.705842, 14828.510000],
+ [0.500000, 1.000000, 256.000000, 1730.222585, 8710.590000],
+ [0.700000, 1024.000000, 1024.000000, 1369.038389, 19597.910000],
+ [0.900000, 64.000000, 64.000000, 1454.543818, 23658.300000],
+ [0.600000, 256.000000, 256.000000, 1353.926401, 10924.180000],
+ [0.900000, 4.000000, 4.000000, 2240.094436, 35811.290000],
+ [0.900000, 1024.000000, 1024.000000, 1521.803360, 36370.860000],
+ [0.900000, 16.000000, 64.000000, 1537.363365, 25713.840000],
+ [1.000000, 1024.000000, 256.000000, 1612.244480, 34376.890000],
+ [0.600000, 1024.000000, 1.000000, 2918.561304, 12676.230000],
+ [0.800000, 1024.000000, 64.000000, 1447.394204, 19376.390000],
+ [1.000000, 16.000000, 4096.000000, 1295.767130, 19752.540000],
+ [0.900000, 16.000000, 4096.000000, 1284.176859, 17665.830000],
+ [0.500000, 64.000000, 64.000000, 1786.323249, 17090.440000],
+ [0.800000, 1.000000, 256.000000, 1590.643012, 13020.470000],
+ [0.600000, 1.000000, 1.000000, 3722.019016, 12581.950000],
+ [0.500000, 64.000000, 16.000000, 2346.008412, 10628.190000],
+ [0.700000, 16.000000, 256.000000, 1419.439444, 10909.080000],
+ [1.000000, 64.000000, 64.000000, 1495.508178, 27974.310000],
+ [0.900000, 256.000000, 16384.000000, 1568.119023, 36540.200000],
+ [0.700000, 256.000000, 1024.000000, 1309.209494, 13202.940000],
+ [1.000000, 256.000000, 16.000000, 1593.476294, 29959.260000],
+ [0.600000, 16.000000, 16384.000000, 1267.214011, 18165.470000],
+ [0.700000, 1024.000000, 4.000000, 1711.022236, 12862.920000],
+ [1.000000, 64.000000, 4096.000000, 1356.962017, 28415.150000],
+ [0.800000, 16.000000, 4.000000, 2175.249552, 21110.880000],
+ [0.900000, 4.000000, 64.000000, 1706.756992, 23604.100000],
+ [1.000000, 256.000000, 1024.000000, 1450.749101, 32559.760000],
+ [1.000000, 1024.000000, 16.000000, 1696.944579, 25716.020000],
+ [0.700000, 1024.000000, 64.000000, 1413.755921, 10896.830000],
+ [0.700000, 4.000000, 16384.000000, 1273.726730, 18109.140000],
+ [0.600000, 4.000000, 4096.000000, 1299.255183, 11151.040000],
+ [0.600000, 4.000000, 1024.000000, 1382.858640, 8897.740000],
+ [0.900000, 1.000000, 1.000000, 2364.737107, 36310.600000],
+ [0.900000, 64.000000, 16384.000000, 1383.792566, 36559.280000],
+ [0.800000, 1.000000, 16.000000, 2271.844459, 16982.230000],
+ [0.700000, 256.000000, 1.000000, 2566.918453, 16962.530000],
+ [0.900000, 256.000000, 1.000000, 1865.777546, 36325.050000],
+ [0.800000, 64.000000, 4096.000000, 1296.580970, 19881.550000],
+ [0.900000, 256.000000, 1024.000000, 1392.699949, 26037.380000],
+ [0.700000, 64.000000, 64.000000, 1505.616635, 10928.860000],
+ [0.900000, 1.000000, 16384.000000, 1287.551166, 23238.140000],
+ [0.600000, 16.000000, 4096.000000, 1285.295193, 11148.520000],
+ [0.500000, 1.000000, 4096.000000, 1329.484556, 8947.940000],
+ [1.000000, 16.000000, 1.000000, 2369.474345, 36322.510000],
+ [0.900000, 256.000000, 256.000000, 1391.712439, 17420.020000],
+ [0.800000, 1024.000000, 1.000000, 1787.887829, 29570.520000],
+ [0.900000, 64.000000, 256.000000, 1358.717854, 17338.630000],
+ [0.800000, 4.000000, 16384.000000, 1273.680399, 22885.930000],
+ [0.800000, 16.000000, 16384.000000, 1277.089752, 24991.040000],
+ [0.600000, 1.000000, 4096.000000, 1317.818664, 11117.430000],
+ [0.800000, 4.000000, 1024.000000, 1343.234780, 15387.070000],
+ [0.500000, 256.000000, 4096.000000, 1274.208686, 11165.100000],
+ [0.800000, 1024.000000, 1024.000000, 1420.742784, 30340.100000],
+ [0.500000, 1024.000000, 16384.000000, 1321.591240, 33868.750000],
+ [0.700000, 1024.000000, 1.000000, 2204.569258, 29469.740000],
+ [0.700000, 1024.000000, 16384.000000, 1524.871729, 36434.350000],
+ [0.600000, 1024.000000, 1024.000000, 1317.814555, 13172.090000],
+ [0.700000, 64.000000, 256.000000, 1359.437760, 13076.980000],
+ [0.900000, 4.000000, 16.000000, 2006.496131, 29844.430000],
+ [0.800000, 1.000000, 4.000000, 2477.453345, 23270.780000],
+ [0.500000, 1024.000000, 64.000000, 1523.244259, 8761.100000],
+ [0.800000, 256.000000, 1.000000, 2077.487352, 29570.100000],
+ [0.600000, 16.000000, 1024.000000, 1334.947039, 11038.730000],
+ [1.000000, 256.000000, 16384.000000, 1697.459511, 36489.610000],
+ [0.800000, 256.000000, 16.000000, 1504.079040, 15143.090000],
+ [0.500000, 4.000000, 1024.000000, 1413.822886, 10958.240000],
+ [0.500000, 64.000000, 4096.000000, 1278.813005, 11103.810000],
+ [0.500000, 1.000000, 64.000000, 2227.201820, 10742.500000],
+ [1.000000, 1.000000, 16384.000000, 1301.175618, 25068.840000],
+ [0.500000, 4.000000, 256.000000, 1643.681520, 10869.120000],
+ [0.800000, 16.000000, 1.000000, 2560.527040, 29376.070000],
+ [0.500000, 16.000000, 1024.000000, 1364.872251, 8879.340000],
+ [0.800000, 16.000000, 64.000000, 1572.391232, 15021.860000],
+ [1.000000, 1024.000000, 1024.000000, 1738.719119, 36385.890000],
+ [0.800000, 1024.000000, 16.000000, 1509.744747, 17226.820000],
+ [0.900000, 1024.000000, 1.000000, 1654.893494, 36341.060000],
+ [1.000000, 16.000000, 1024.000000, 1335.264417, 15416.250000],
+ [0.900000, 256.000000, 64.000000, 1453.060437, 21542.420000],
+ [0.700000, 16.000000, 64.000000, 1668.761813, 10853.180000],
+ [0.800000, 16.000000, 1024.000000, 1309.582372, 15377.860000],
+ [0.600000, 256.000000, 1.000000, 3294.021457, 12682.510000],
+ [1.000000, 1024.000000, 4096.000000, 1847.090224, 36369.090000],
+ [0.700000, 64.000000, 16.000000, 1784.594764, 10797.140000],
+ [0.900000, 1.000000, 256.000000, 1582.721013, 17274.740000],
+ [0.700000, 256.000000, 256.000000, 1343.241802, 13073.810000],
+ [0.500000, 64.000000, 256.000000, 1457.530339, 10908.290000],
+ [0.600000, 1024.000000, 64.000000, 1438.727854, 8780.440000],
+ [0.600000, 256.000000, 4.000000, 2360.980123, 18943.070000],
+ [0.600000, 4.000000, 64.000000, 1943.349721, 10784.510000],
+ [0.900000, 16.000000, 16384.000000, 1282.164335, 27758.470000],
+ [1.000000, 4.000000, 4.000000, 2306.935465, 36328.220000],
+ [0.500000, 256.000000, 16.000000, 2088.040327, 10697.110000],
+ [0.600000, 4.000000, 16.000000, 2482.891469, 10661.270000],
+ [0.700000, 64.000000, 1.000000, 2810.881943, 27147.190000],
+ [0.700000, 4.000000, 16.000000, 2259.310536, 12780.130000],
+ [0.900000, 256.000000, 4.000000, 1601.761693, 36219.240000],
+ [0.500000, 64.000000, 4.000000, 3198.956320, 12589.520000],
+ [0.800000, 1.000000, 64.000000, 1917.207978, 14986.340000],
+ [0.600000, 1.000000, 16384.000000, 1272.839987, 16139.940000],
+ [0.500000, 16.000000, 4.000000, 3346.584734, 22882.000000],
+ [0.900000, 1024.000000, 4.000000, 1513.696030, 31977.800000],
+ [1.000000, 4.000000, 1024.000000, 1358.437431, 19614.580000],
+ [1.000000, 1.000000, 16.000000, 2244.860585, 36330.680000],
+ [1.000000, 256.000000, 1.000000, 1855.132826, 36336.080000],
+ [0.900000, 256.000000, 16.000000, 1509.932842, 27889.120000],
+ [1.000000, 16.000000, 16384.000000, 1302.230813, 27616.620000],
+ [0.500000, 4.000000, 1.000000, 5149.513805, 12552.220000],
+ [1.000000, 64.000000, 1.000000, 2109.894998, 36349.640000],
+ [0.600000, 64.000000, 1.000000, 3619.232213, 12633.430000],
+ [0.800000, 256.000000, 4.000000, 1713.158518, 23433.050000],
+ [0.900000, 64.000000, 1024.000000, 1318.388432, 17534.690000],
+ [0.700000, 256.000000, 16.000000, 1594.416105, 10857.250000],
+ [0.600000, 1.000000, 4.000000, 3126.891155, 18807.430000],
+ [0.600000, 4.000000, 16384.000000, 1270.425851, 16183.590000],
+ [0.800000, 16.000000, 256.000000, 1385.504627, 17284.460000],
+ [1.000000, 64.000000, 256.000000, 1388.593292, 19561.620000],
+ [0.800000, 256.000000, 256.000000, 1347.624790, 17293.180000],
+ [0.600000, 16.000000, 64.000000, 1786.518680, 10804.840000],
+ [0.900000, 64.000000, 4096.000000, 1318.135676, 21929.520000],
+ [0.500000, 1.000000, 1.000000, 5258.112826, 12536.720000],
+ [0.600000, 1.000000, 256.000000, 1660.049798, 8731.610000],
+ [0.700000, 64.000000, 1024.000000, 1303.411361, 13189.740000],
+ [0.900000, 1.000000, 16.000000, 2210.134257, 29606.270000],
+ [0.500000, 1024.000000, 4096.000000, 1296.897775, 15553.910000],
+ [0.600000, 4.000000, 1.000000, 3747.871323, 12555.700000],
+ [0.500000, 16.000000, 1.000000, 5229.181160, 12538.960000],
+ [0.500000, 1024.000000, 16.000000, 1880.399470, 10733.760000],
+ [0.700000, 256.000000, 4096.000000, 1314.937940, 19874.800000],
+ [0.900000, 1024.000000, 16.000000, 1586.647560, 21473.500000],
+ [1.000000, 16.000000, 256.000000, 1404.940141, 21758.270000],
+ [0.700000, 1.000000, 256.000000, 1619.226174, 10852.560000],
+ [0.800000, 256.000000, 64.000000, 1408.231244, 19407.500000],
+ [0.900000, 4.000000, 16384.000000, 1280.796294, 25147.340000],
+ [0.800000, 256.000000, 1024.000000, 1330.546637, 19667.980000],
+ [0.500000, 4.000000, 16384.000000, 1270.589627, 13865.620000],
+ [0.700000, 16.000000, 4096.000000, 1281.538285, 13378.880000],
+ [0.800000, 64.000000, 16384.000000, 1320.508949, 34243.490000],
+ [0.900000, 1.000000, 1024.000000, 1405.254246, 17379.910000],
+ [0.900000, 16.000000, 1.000000, 2341.462081, 36329.320000],
+ [0.700000, 64.000000, 16384.000000, 1282.492457, 27080.430000],
+ [0.700000, 256.000000, 4.000000, 1951.044824, 12853.060000],
+ [1.000000, 1.000000, 256.000000, 1593.512913, 21558.680000],
+ [1.000000, 256.000000, 4096.000000, 1588.528832, 36406.420000],
+ [0.900000, 1.000000, 4.000000, 2429.684277, 33713.730000],
+ [0.500000, 1.000000, 16384.000000, 1271.753365, 13760.600000],
+ [1.000000, 1024.000000, 64.000000, 1579.061710, 25761.850000],
+ [0.700000, 16.000000, 4.000000, 2487.987431, 18947.020000],
+ [0.500000, 4.000000, 4096.000000, 1309.276414, 8971.530000],
+ [0.800000, 4.000000, 4096.000000, 1289.574825, 15376.110000],
+ [0.900000, 4.000000, 4096.000000, 1294.339616, 17704.140000],
+ [0.500000, 1024.000000, 1024.000000, 1302.023694, 10982.080000],
+ [0.600000, 1024.000000, 256.000000, 1351.998671, 10947.490000],
+ [0.500000, 1024.000000, 4.000000, 2542.238814, 29202.250000],
+ [0.800000, 4.000000, 256.000000, 1478.045732, 15249.950000],
+ [1.000000, 4.000000, 4096.000000, 1304.324411, 19810.090000],
+ [0.900000, 64.000000, 1.000000, 2118.723891, 36320.250000],
+ [0.600000, 64.000000, 16.000000, 2033.429989, 10707.310000],
+ [1.000000, 1.000000, 64.000000, 1913.104369, 27754.330000],
+ [0.500000, 256.000000, 1024.000000, 1307.561280, 11039.350000],
+ [0.600000, 1024.000000, 16384.000000, 1393.614637, 36791.210000],
+ [1.000000, 16.000000, 16.000000, 1750.590736, 36267.290000],
+ [0.500000, 256.000000, 64.000000, 1626.654117, 14953.820000],
+ [0.900000, 256.000000, 4096.000000, 1439.558928, 36385.390000],
+ [0.800000, 1.000000, 1.000000, 2548.705258, 29322.290000],
+ [0.700000, 1024.000000, 256.000000, 1363.001016, 15230.860000],
+ [0.900000, 4.000000, 256.000000, 1463.333299, 19454.290000],
+ [0.800000, 1.000000, 16384.000000, 1281.177353, 20554.140000],
+ [0.600000, 1.000000, 64.000000, 2086.676125, 10750.130000],
+ [1.000000, 1.000000, 4096.000000, 1323.445870, 21980.400000],
+ [0.700000, 4.000000, 4.000000, 2711.405961, 12692.900000],
+ [0.600000, 256.000000, 1024.000000, 1302.341384, 11066.980000],
+ [0.700000, 1.000000, 16384.000000, 1276.738560, 18358.530000],
+ [0.600000, 256.000000, 64.000000, 1501.526137, 8766.400000],
+ [0.700000, 4.000000, 64.000000, 1843.379712, 10832.460000],
+ [0.500000, 256.000000, 256.000000, 1401.072001, 6741.450000],
+ [0.700000, 1.000000, 4.000000, 2729.267232, 23222.120000],
+ [0.800000, 4.000000, 4.000000, 2384.413805, 23341.310000],
+ [0.600000, 256.000000, 16384.000000, 1310.790773, 32127.290000],
+ [0.600000, 64.000000, 1024.000000, 1309.290893, 10988.550000],
+ [0.600000, 4.000000, 256.000000, 1568.538127, 10911.060000],
+ [0.700000, 64.000000, 4096.000000, 1276.161920, 15525.330000],
+ [0.700000, 1024.000000, 4096.000000, 1397.052266, 36423.470000],
+ [0.800000, 16.000000, 4096.000000, 1280.470261, 15551.190000],
+ [0.800000, 1.000000, 1024.000000, 1406.231488, 13228.600000],
+ [0.600000, 16.000000, 256.000000, 1473.258879, 10886.050000],
+ [0.800000, 64.000000, 1024.000000, 1304.882655, 15346.050000],
+ [0.700000, 4.000000, 4096.000000, 1291.760503, 13324.520000],
+ [0.700000, 1.000000, 16.000000, 2422.500695, 14833.800000],
+ [0.800000, 64.000000, 1.000000, 2347.846308, 29466.530000],
+ [0.900000, 4.000000, 1.000000, 2452.668607, 36313.550000],
+ [0.500000, 1024.000000, 64.000000, 1520.064450, 10877.800000],
+ [0.800000, 1024.000000, 256.000000, 1389.175687, 17386.800000],
+ [0.700000, 256.000000, 16384.000000, 1377.931876, 36591.050000],
+ [0.500000, 1024.000000, 1.000000, 4432.511510, 12621.590000],
+ [0.900000, 16.000000, 256.000000, 1381.783824, 21799.460000],
+ [0.900000, 4.000000, 1024.000000, 1345.175995, 17429.470000],
+ [0.600000, 1024.000000, 16.000000, 1626.743743, 12900.210000],
+ [0.900000, 64.000000, 4.000000, 1789.856352, 31967.840000],
+ [0.600000, 256.000000, 4096.000000, 1290.394505, 17664.880000],
+ [1.000000, 256.000000, 64.000000, 1503.488973, 21555.060000],
+ [1.000000, 4.000000, 1.000000, 2562.137775, 36323.630000],
+ [1.000000, 1.000000, 1024.000000, 1416.648457, 19595.790000],
+ [0.700000, 1.000000, 1.000000, 2975.404068, 16789.480000],
+ [0.500000, 16.000000, 256.000000, 1549.807417, 10858.790000],
+ [0.700000, 16.000000, 1.000000, 2961.509740, 27084.010000],
+ [0.900000, 1024.000000, 16384.000000, 1929.981674, 36384.180000],
+ [1.000000, 1024.000000, 1.000000, 1693.890539, 36343.210000],
+ [0.900000, 1.000000, 4096.000000, 1314.611328, 17626.750000],
+ [0.500000, 16.000000, 16.000000, 2612.878741, 10712.800000],
+ [0.500000, 4.000000, 4.000000, 3481.699285, 29060.450000],
+ [0.500000, 256.000000, 4.000000, 2939.348266, 12609.960000],
+ [0.700000, 4.000000, 1024.000000, 1360.104102, 10955.300000],
+ [0.500000, 1.000000, 1024.000000, 1460.865658, 8810.670000],
+ [0.800000, 1024.000000, 16384.000000, 1670.162786, 36825.090000],
+ [0.500000, 16.000000, 64.000000, 1969.256740, 6563.080000],
+ [0.700000, 1024.000000, 16.000000, 1505.384479, 10848.440000],
+ [0.500000, 64.000000, 1024.000000, 1329.959506, 8891.580000],
+ [0.800000, 256.000000, 16384.000000, 1442.360685, 36395.860000],
+ [0.700000, 16.000000, 16.000000, 2035.164063, 12877.330000],
+ [0.700000, 16.000000, 1024.000000, 1318.230944, 11043.220000],
+ [0.800000, 64.000000, 4.000000, 1943.102596, 27575.300000],
+ [0.700000, 64.000000, 4.000000, 2250.809494, 12793.650000],
+ [0.600000, 64.000000, 4096.000000, 1271.714717, 13322.590000],
+ [0.600000, 16.000000, 1.000000, 3770.110326, 12621.170000],
+ [1.000000, 256.000000, 256.000000, 1417.792032, 21866.210000],
+ [1.000000, 256.000000, 4.000000, 1620.128845, 36348.480000],
+ [0.600000, 64.000000, 256.000000, 1395.590972, 10909.370000],
+ [0.700000, 1.000000, 1024.000000, 1414.752908, 10989.010000],
+ [1.000000, 64.000000, 4.000000, 1812.145883, 36335.070000],
+ [0.700000, 4.000000, 256.000000, 1521.322055, 10894.450000],
+ [0.500000, 16.000000, 4096.000000, 1292.698578, 9022.760000],
+ [0.500000, 1.000000, 16.000000, 2836.012122, 16765.670000],
+ [1.000000, 64.000000, 16384.000000, 1433.882412, 36474.950000],
+ [0.800000, 64.000000, 64.000000, 1454.028740, 17186.180000],
+ [0.800000, 4.000000, 64.000000, 1752.565517, 15054.510000],
+ [0.600000, 16.000000, 16.000000, 2277.597145, 10683.480000]])
+
+ kappa_values = (0.5, 0.6, 0.7, 0.8, 0.9, 1.0)
+ tau_values = (1.0, 4.0, 16.0, 64.0, 256.0, 1024.0)
+ s_values = (1.0, 4.0, 16.0, 64.0, 256.0, 1024.0, 4096.0, 16384.0)
+
+ # Convert the array into an easily accessible tree
+ config_tree = dict()
+ for kappa_value in kappa_values:
+ config_tree[kappa_value] = dict()
+ for Tau_value in tau_values:
+ config_tree[kappa_value][Tau_value] = dict()
+ for S_value in s_values:
+ config_tree[kappa_value][Tau_value][S_value] = [np.inf, np.inf]
+
+ # Populate the tree
+ for config in configurations:
+ config_tree[config[0]][config[1]][config[2]][0] = config[3]
+ config_tree[config[0]][config[1]][config[2]][1] = config[4]
+
+ print kappa, tau, s
+ if ret_time:
+ return config_tree[kappa_values[kappa]][tau_values[tau]][s_values[s]][1]
+ else:
+ return config_tree[kappa_values[kappa]][tau_values[tau]][s_values[s]][0]
+
+
+def save_svm_on_grid(params, ret_time=False, **kwargs):
+ if "C" not in params or "alpha" not in params or "epsilon" not in params:
+ sys.stderr.write("No params found ['C', 'alpha', 'epsilon']: %s\n" % str(params))
+ return np.NaN
+ c = int(float(params["C"]))
+ alpha = int(float(params["alpha"]))
+ epsilon = int(float(params["epsilon"]))
+
+ if type(c) == np.ndarray:
+ c = c[0]
+ alpha = alpha[0]
+ epsilon = epsilon[0]
+ return svm_on_grid(c=c, alpha=alpha, epsilon=epsilon, ret_time=ret_time)
+
+
+def svm_on_grid(c, alpha, epsilon, ret_time=False):
+ # Values for an 6*8*8 grid search which was performed by Hofman et. al.
+ # Values obtained from Jasper Snoek
+ # C, alpha, epsilon
+ # opt_time: return time instead of performance
+ svm_grid_configs = [[600, 0.5, 0.01, 0.2762, 191.98],
+ [900, 0.8, 0.1, 0.2742, 99.976],
+ [2000, 5, 0.1, 0.2791, 114.41],
+ [5000, 1, 0.0001, 0.2577, 748.726],
+ [1, 5, 0.0001, 0.28714, 1048.862],
+ [600, 0.8, 0.1, 0.27926, 93.102],
+ [75, 0.5, 0.0001, 0.33702, 843.996],
+ [10000, 1.5, 0.01, 0.26462, 297.46],
+ [1000, 0.8, 0.0001, 0.2544, 1986.86666667],
+ [50, 0.2, 0.01, 0.39706, 226.594],
+ [5000, 3, 0.001, 0.26474, 814.784],
+ [1000000, 1, 0.01, 0.5, 156.472],
+ [10, 5, 0.0001, 0.28722, 1098.516],
+ [5, 3, 0.001, 0.34864, 305.692],
+ [10000, 3, 0.001, 0.26552, 873.812],
+ [800, 1, 0.001, 0.2785, 347.486],
+ [100, 0.5, 0.0001, 0.3232, 833.83],
+ [500, 1, 0.001, 0.2821, 381.762],
+ [300, 2, 0.001, 0.288, 312.296],
+ [600, 1.5, 0.01, 0.28258, 195.106],
+ [2000, 0.6, 0.01, 0.27058, 161.32],
+ [100, 0.5, 0.1, 0.35676, 79.6],
+ [400, 0.3, 0.01, 0.28054, 207.808],
+ [5000, 3, 0.01, 0.27486, 239.416],
+ [1000, 2, 0.01, 0.28104, 155.44],
+ [100, 0.1, 0.001, 0.34946, 482.008],
+ [10, 0.8, 0.001, 0.38166, 424.346],
+ [10, 3, 0.001, 0.31004, 475.116],
+ [2000, 0.4, 0.0001, 0.24785, 1623.4025],
+ [25, 0.2, 0.001, 0.41688, 411.466],
+ [3000, 0.4, 0.0001, 0.24756, 1248.566],
+ [600, 0.4, 0.0001, 0.27546, 857.812],
+ [4000, 5, 0.001, 0.26938, 831.988],
+ [1000, 0.9, 0.001, 0.27682, 305.498],
+ [7000, 5, 0.001, 0.26876, 793.69],
+ [600, 0.2, 0.01, 0.27266, 191.62],
+ [700, 0.8, 0.001, 0.27844, 330.508],
+ [7000, 3, 0.001, 0.26466, 848.348],
+ [75, 0.3, 0.0001, 0.35082, 908.324],
+ [1000000, 0.8, 0.1, 0.5, 90.708],
+ [5, 0.7, 0.001, 0.39864, 364.084],
+ [7000, 2, 0.001, 0.26008, 833.362],
+ [1, 0.6, 0.01, 0.42818, 173.37],
+ [7000, 0.1, 0.01, 0.25052, 209.178],
+ [500, 0.7, 0.01, 0.28004, 190.85],
+ [3000, 0.7, 0.001, 0.25062, 685.636],
+ [4000, 0.2, 0.0001, 0.24396, 1086.718],
+ [50, 0.5, 0.001, 0.35852, 425.03],
+ [900, 0.7, 0.001, 0.27574, 297.056],
+ [400, 2, 0.01, 0.28514, 178.252],
+ [5000, 0.1, 0.0001, 0.24196, 939.238],
+ [200, 0.6, 0.0001, 0.29602, 824.136],
+ [50, 0.5, 0.0001, 0.356, 1033.77],
+ [3000, 0.9, 0.01, 0.27026, 170.61],
+ [300, 3, 0.01, 0.28716, 178.034],
+ [0.1, 5, 0.01, 0.30022, 168.136],
+ [7000, 0.4, 0.001, 0.2464, 597.92],
+ [1000000, 0.8, 0.0001, 0.5, 945.362],
+ [200, 0.1, 0.0001, 0.29966, 837.446],
+ [6000, 1, 0.0001, 0.26336, 480.288],
+ [75, 1, 0.01, 0.33582, 186.376],
+ [10000, 0.4, 0.0001, 0.2461, 1121.432],
+ [600, 0.9, 0.001, 0.28038, 377.958],
+ [700, 0.8, 0.1, 0.27686, 89.688],
+ [100, 0.4, 0.0001, 0.32776, 854.29],
+ [6000, 3, 0.1, 0.27414, 159.082],
+ [5000, 0.6, 0.01, 0.26246, 207.476],
+ [3000, 0.1, 0.0001, 0.24226, 1101.012],
+ [7000, 0.7, 0.01, 0.24986, 359.704],
+ [100, 0.9, 0.1, 0.31914, 79.928],
+ [400, 0.5, 0.001, 0.28194, 398.602],
+ [1000, 5, 0.001, 0.26784, 785.06],
+ [1, 1.5, 0.0001, 0.34782, 1062.154],
+ [800, 0.7, 0.001, 0.27648, 309.812],
+ [4000, 0.2, 0.001, 0.24368, 645.686],
+ [7000, 0.6, 0.1, 0.26214, 150.098],
+ [1000000, 0.9, 0.001, 0.5, 280.154],
+ [5, 5, 0.1, 0.33336, 66.22],
+ [1, 1.5, 0.1, 0.39298, 68.592],
+ [4000, 1, 0.1, 0.27002, 95.75],
+ [2000, 0.9, 0.1, 0.27118, 101.142],
+ [900, 3, 0.1, 0.28126, 97.174],
+ [1000, 2, 0.0001, 0.260866666667, 1403.96],
+ [10000, 1.5, 0.0001, 0.2825, 1146.192],
+ [5000, 0.2, 0.1, 0.25934, 115.906],
+ [4000, 0.6, 0.001, 0.2495, 625.496],
+ [500, 2, 0.0001, 0.26514, 1913.09],
+ [2000, 0.9, 0.0001, 0.252625, 1501.0925],
+ [500, 0.3, 0.001, 0.27694, 401.348],
+ [2000, 0.2, 0.1, 0.26326, 98.796],
+ [1000000, 0.2, 0.1, 0.5, 114.502],
+ [300, 0.1, 0.001, 0.28604, 390.852],
+ [50, 0.4, 0.001, 0.36676, 441.684],
+ [100, 5, 0.0001, 0.27402, 1192.626],
+ [800, 2, 0.0001, 0.26402, 1703.954],
+ [1, 5, 0.01, 0.29758, 183.424],
+ [5000, 0.9, 0.001, 0.25164, 777.066],
+ [2000, 2, 0.001, 0.2608, 711.664],
+ [600, 1, 0.1, 0.28378, 54.028],
+ [4000, 2, 0.0001, 0.26076, 1192.924],
+ [25, 0.7, 0.001, 0.36862, 389.97],
+ [500, 0.3, 0.1, 0.27662, 86.12],
+ [4000, 0.4, 0.01, 0.26418, 163.332],
+ [500, 0.9, 0.1, 0.28126, 84.918],
+ [900, 0.5, 0.01, 0.27312, 187.244],
+ [10000, 1.5, 0.1, 0.27406, 148.158],
+ [900, 1.5, 0.01, 0.28038, 159.878],
+ [600, 0.3, 0.0001, 0.2742, 843.632],
+ [600, 0.1, 0.001, 0.27114, 399.358],
+ [200, 0.9, 0.01, 0.29442, 187.706],
+ [50, 0.7, 0.01, 0.35294, 198.696],
+ [25, 0.2, 0.1, 0.46938, 72.34],
+ [300, 1.5, 0.0001, 0.28738, 821.884],
+ [75, 0.7, 0.001, 0.32858, 419.148],
+ [800, 0.4, 0.01, 0.27268, 188.818],
+ [7000, 0.8, 0.01, 0.25148, 369.136],
+ [6000, 1.5, 0.1, 0.26684, 140.058],
+ [3000, 0.9, 0.1, 0.26778, 113.164],
+ [3000, 3, 0.001, 0.26436, 781.674],
+ [0.1, 1, 0.01, 0.38922, 150.924],
+ [100, 0.1, 0.01, 0.35448, 223.032],
+ [1000000, 0.5, 0.01, 0.5, 180.966],
+ [1, 0.4, 0.0001, 0.44414, 1033.23],
+ [600, 0.3, 0.1, 0.27302, 86.148],
+ [10, 5, 0.1, 0.3787, 65.432],
+ [3000, 5, 0.0001, 0.2695, 1539.228],
+ [4000, 3, 0.01, 0.2769, 191.418],
+ [100, 1.5, 0.1, 0.30546, 84.802],
+ [6000, 0.2, 0.01, 0.25666, 213.316],
+ [0.1, 0.3, 0.1, 0.47566, 62.724],
+ [200, 0.2, 0.001, 0.2981, 379.944],
+ [800, 0.6, 0.0001, 0.27498, 697.262],
+ [1, 0.1, 0.001, 0.46926, 342.85],
+ [4000, 0.6, 0.1, 0.26052, 116.586],
+ [800, 1, 0.0001, 0.27768, 672.318],
+ [4000, 0.5, 0.001, 0.24794, 633.722],
+ [3000, 0.2, 0.001, 0.25498, 463.4],
+ [10000, 0.8, 0.1, 0.26354, 141.786],
+ [1000, 0.5, 0.1, 0.2715, 96.61],
+ [500, 0.9, 0.001, 0.2818, 388.982],
+ [900, 0.5, 0.0001, 0.272566666667, 636.433333333],
+ [1000000, 0.7, 0.01, 0.5, 171.13],
+ [5000, 0.4, 0.001, 0.24662, 696.42],
+ [300, 0.1, 0.01, 0.28604, 188.872],
+ [1000, 0.4, 0.01, 0.27098, 179.646],
+ [25, 0.7, 0.0001, 0.36802, 831.36],
+ [1000000, 2, 0.1, 0.47274, 81.712],
+ [1000000, 0.3, 0.1, 0.5, 99.086],
+ [800, 0.1, 0.001, 0.26876, 303.86],
+ [25, 1.5, 0.001, 0.39774, 275.812],
+ [6000, 0.5, 0.0001, 0.24768, 1200.336],
+ [1000000, 2, 0.01, 0.40806, 138.358],
+ [0.1, 0.1, 0.01, 0.46946, 168.158],
+ [500, 2, 0.01, 0.28406, 180.38],
+ [600, 1, 0.001, 0.28062, 367.558],
+ [400, 0.4, 0.0001, 0.28092, 924.564],
+ [50, 0.4, 0.1, 0.44714, 64.674],
+ [300, 0.9, 0.1, 0.2877, 89.732],
+ [200, 2, 0.01, 0.29112, 174.154],
+ [1000000, 0.9, 0.1, 0.5, 97.332],
+ [600, 0.8, 0.01, 0.27912, 184.422],
+ [10, 0.2, 0.1, 0.48226, 56.632],
+ [2000, 1.5, 0.0001, 0.25795, 1302.53],
+ [6000, 0.8, 0.001, 0.25066, 653.718],
+ [5000, 0.8, 0.001, 0.251, 645.086],
+ [1, 0.1, 0.1, 0.48462, 64.426],
+ [100, 1.5, 0.001, 0.30438, 383.798],
+ [1, 0.5, 0.01, 0.43472, 177.06],
+ [1000000, 3, 0.1, 0.45206, 90.21],
+ [500, 1.5, 0.001, 0.28366, 376.972],
+ [7000, 0.7, 0.1, 0.2608, 137.904],
+ [300, 0.8, 0.0001, 0.28664, 897.056],
+ [900, 5, 0.01, 0.28444, 159.834],
+ [500, 0.2, 0.0001, 0.2753, 914.754],
+ [1, 0.2, 0.001, 0.4602, 344.65],
+ [7000, 2, 0.01, 0.26318, 340.2],
+ [7000, 0.9, 0.001, 0.25146, 679.86],
+ [50, 0.3, 0.001, 0.37742, 429.536],
+ [800, 0.7, 0.0001, 0.27596, 734.622],
+ [1, 0.4, 0.1, 0.45824, 72.28],
+ [300, 0.4, 0.1, 0.28392, 90.676],
+ [3000, 0.3, 0.01, 0.26454, 154.638],
+ [7000, 1.5, 0.01, 0.25772, 409.496],
+ [300, 0.1, 0.0001, 0.28594, 831.236],
+ [200, 0.7, 0.01, 0.29622, 188.1],
+ [5, 0.2, 0.0001, 0.4509, 974.506],
+ [1000, 0.9, 0.1, 0.2737, 88.956],
+ [6000, 1.5, 0.001, 0.257, 686.404],
+ [700, 0.9, 0.1, 0.27722, 89.512],
+ [400, 0.5, 0.0001, 0.28146, 955.994],
+ [4000, 0.2, 0.1, 0.25786, 113.644],
+ [500, 0.8, 0.0001, 0.28066, 924.76],
+ [10, 0.7, 0.01, 0.3929, 176.214],
+ [5, 0.8, 0.01, 0.39682, 164.94],
+ [1000000, 1.5, 0.1, 0.47812, 82.882],
+ [700, 0.1, 0.1, 0.2669, 88.556],
+ [100, 0.8, 0.001, 0.3146, 405.786],
+ [5, 0.6, 0.1, 0.43958, 64.786],
+ [25, 3, 0.01, 0.30488, 183.214],
+ [400, 1.5, 0.01, 0.2845, 185.428],
+ [50, 1.5, 0.0001, 0.31074, 1144.406],
+ [10000, 5, 0.001, 0.27124, 915.448],
+ [900, 1.5, 0.001, 0.28028, 300.058],
+ [100, 1.5, 0.0001, 0.30012, 1052.118],
+ [10000, 2, 0.0001, 0.28056, 1245.674],
+ [6000, 3, 0.0001, 0.26562, 1629.002],
+ [500, 5, 0.001, 0.27102, 742.192],
+ [600, 3, 0.0001, 0.26594, 1653.042],
+ [4000, 0.7, 0.1, 0.2623, 118.162],
+ [10000, 5, 0.01, 0.27556, 254.364],
+ [800, 5, 0.1, 0.28192, 91.156],
+ [300, 5, 0.001, 0.27646, 590.67],
+ [400, 0.2, 0.01, 0.27976, 198.17],
+ [700, 0.1, 0.0001, 0.269225, 717.565],
+ [2000, 0.4, 0.01, 0.26748, 159.678],
+ [0.1, 1.5, 0.0001, 0.3572, 1120.41],
+ [6000, 0.2, 0.001, 0.2437, 562.952],
+ [10, 2, 0.0001, 0.32, 1124.784],
+ [50, 0.5, 0.1, 0.45302, 63.464],
+ [400, 3, 0.0001, 0.26644, 1752.45],
+ [0.1, 0.5, 0.0001, 0.4376, 1166.87],
+ [1000, 3, 0.01, 0.2825, 162.89],
+ [7000, 0.1, 0.1, 0.255, 123.884],
+ [50, 0.6, 0.01, 0.35986, 200.666],
+ [700, 0.2, 0.0001, 0.27095, 727.81],
+ [5, 3, 0.01, 0.31416, 161.456],
+ [2000, 0.8, 0.001, 0.26098, 567.132],
+ [75, 1.5, 0.1, 0.32812, 81.152],
+ [3000, 1, 0.1, 0.27202, 90.778],
+ [400, 1.5, 0.1, 0.28474, 84.954],
+ [200, 0.1, 0.001, 0.30018, 382.706],
+ [4000, 0.1, 0.0001, 0.24168, 997.292],
+ [700, 0.8, 0.0001, 0.27715, 755.7525],
+ [1000000, 0.6, 0.1, 0.5, 88.332],
+ [5, 0.1, 0.1, 0.4876, 59.532],
+ [10000, 1.5, 0.001, 0.25714, 728.428],
+ [300, 0.7, 0.001, 0.28744, 360.1],
+ [25, 0.4, 0.01, 0.40768, 206.86],
+ [1000000, 0.8, 0.01, 0.5, 137.506],
+ [75, 0.4, 0.0001, 0.3432, 884.06],
+ [10000, 0.8, 0.01, 0.2555, 319.836],
+ [10000, 0.7, 0.0001, 0.27342, 1126.338],
+ [800, 2, 0.1, 0.2803, 85.47],
+ [1000000, 5, 0.001, 0.40736, 313.006],
+ [6000, 3, 0.001, 0.26476, 801.316],
+ [6000, 0.5, 0.1, 0.25866, 127.438],
+ [75, 0.9, 0.0001, 0.3195, 796.796],
+ [1000000, 3, 0.01, 0.3743, 152.658],
+ [800, 0.2, 0.01, 0.27004, 186.514],
+ [6000, 0.7, 0.001, 0.24972, 639.95],
+ [10000, 0.5, 0.1, 0.26096, 134.292],
+ [700, 1.5, 0.0001, 0.264075, 1922.0675],
+ [50, 1, 0.01, 0.35354, 200.424],
+ [500, 0.6, 0.1, 0.277225, 90.7025],
+ [1, 0.4, 0.001, 0.44334, 375.616],
+ [400, 5, 0.0001, 0.26774, 1558.12],
+ [10, 0.2, 0.01, 0.44462, 195.0],
+ [300, 3, 0.0001, 0.26762, 1884.616],
+ [50, 1.5, 0.1, 0.38136, 68.446],
+ [200, 5, 0.01, 0.28722, 146.236],
+ [700, 2, 0.0001, 0.2646, 1563.2175],
+ [3000, 5, 0.1, 0.27902, 132.058],
+ [600, 0.9, 0.0001, 0.27958, 866.892],
+ [75, 0.1, 0.001, 0.37422, 442.79],
+ [0.1, 3, 0.0001, 0.31042, 1130.442],
+ [400, 0.9, 0.001, 0.28388, 401.908],
+ [3000, 3, 0.1, 0.27484, 123.96],
+ [100, 0.8, 0.0001, 0.3133, 860.986],
+ [100, 1, 0.01, 0.32508, 182.17],
+ [100, 0.6, 0.001, 0.32034, 408.704],
+ [10, 0.3, 0.1, 0.46998, 61.478],
+ [6000, 0.6, 0.01, 0.25478, 272.768],
+ [0.1, 1.5, 0.1, 0.39805, 64.5525],
+ [1000, 0.6, 0.001, 0.27398, 298.708],
+ [6000, 0.2, 0.1, 0.2544, 120.432],
+ [7000, 1.5, 0.1, 0.26866, 146.586],
+ [25, 0.1, 0.1, 0.47186, 67.134],
+ [0.1, 0.8, 0.0001, 0.40828, 1117.866],
+ [1000, 3, 0.0001, 0.263733333333, 1483.86666667],
+ [3000, 0.6, 0.0001, 0.24924, 1198.446],
+ [5000, 2, 0.01, 0.27108, 212.174],
+ [300, 0.2, 0.01, 0.28646, 194.702],
+ [200, 5, 0.001, 0.28724, 355.718],
+ [500, 0.7, 0.1, 0.27752, 89.834],
+ [500, 1, 0.0001, 0.2824, 758.5],
+ [800, 1.5, 0.001, 0.281, 311.938],
+ [7000, 0.8, 0.1, 0.26344, 147.49],
+ [6000, 0.2, 0.0001, 0.2437, 916.334],
+ [5000, 0.7, 0.001, 0.24986, 623.874],
+ [25, 0.2, 0.0001, 0.41636, 913.428],
+ [600, 3, 0.01, 0.28424, 188.116],
+ [25, 0.4, 0.1, 0.4509, 67.05],
+ [900, 3, 0.0001, 0.2641, 1598.26333333],
+ [600, 0.2, 0.0001, 0.2729, 862.466],
+ [5, 1.5, 0.001, 0.34766, 358.646],
+ [0.1, 0.2, 0.01, 0.46206, 174.574],
+ [400, 0.2, 0.001, 0.2799, 396.37],
+ [400, 3, 0.1, 0.28594, 87.66],
+ [1000, 0.5, 0.001, 0.27266, 296.31],
+ [2000, 1.5, 0.01, 0.27668, 165.654],
+ [1000, 0.6, 0.1, 0.2718, 93.558],
+ [0.1, 1, 0.1, 0.44262, 48.88],
+ [7000, 0.4, 0.0001, 0.24664, 1141.41],
+ [7000, 3, 0.1, 0.27498, 157.584],
+ [200, 0.8, 0.01, 0.29548, 187.16],
+ [50, 5, 0.01, 0.29194, 176.454],
+ [3000, 0.8, 0.0001, 0.25168, 1197.344],
+ [6000, 0.1, 0.001, 0.2411, 534.628],
+ [75, 0.6, 0.0001, 0.3322, 859.47],
+ [75, 0.5, 0.1, 0.38742, 77.122],
+ [2000, 0.5, 0.0001, 0.249, 1647.165],
+ [100, 1.5, 0.01, 0.30422, 188.836],
+ [3000, 1, 0.01, 0.27126, 143.914],
+ [4000, 0.1, 0.01, 0.25918, 167.734],
+ [900, 0.6, 0.0001, 0.2736, 716.086666667],
+ [5, 0.9, 0.01, 0.38836, 164.888],
+ [10, 0.5, 0.001, 0.40544, 412.39],
+ [75, 5, 0.0001, 0.276, 1322.574],
+ [2000, 0.7, 0.001, 0.2669, 443.006],
+ [1000, 5, 0.0001, 0.2687, 1425.24],
+ [6000, 0.3, 0.01, 0.25478, 238.268],
+ [25, 1.5, 0.1, 0.37726, 72.872],
+ [50, 0.3, 0.01, 0.38228, 240.814],
+ [1000, 2, 0.001, 0.2768, 372.22],
+ [25, 0.4, 0.0001, 0.39388, 857.742],
+ [75, 0.9, 0.1, 0.34776, 76.166],
+ [25, 1, 0.1, 0.41652, 52.644],
+ [6000, 0.8, 0.1, 0.26438, 130.272],
+ [5000, 0.5, 0.01, 0.2592, 233.634],
+ [10, 0.6, 0.0001, 0.40212, 1154.298],
+ [3000, 1.5, 0.001, 0.25758, 723.972],
+ [10, 0.1, 0.1, 0.4944, 53.742],
+ [1000, 1.5, 0.1, 0.27826, 92.22],
+ [900, 0.5, 0.001, 0.27328, 303.928],
+ [500, 0.9, 0.01, 0.28106, 190.942],
+ [1000, 0.5, 0.01, 0.27242, 185.06],
+ [1000, 1, 0.1, 0.27934, 78.268],
+ [5, 3, 0.1, 0.35928, 61.924],
+ [400, 0.3, 0.001, 0.28038, 400.058],
+ [700, 5, 0.1, 0.28282, 93.462],
+ [50, 0.1, 0.01, 0.4098, 221.192],
+ [7000, 0.2, 0.0001, 0.24358, 871.402],
+ [800, 3, 0.0001, 0.26532, 1699.988],
+ [7000, 3, 0.0001, 0.26776, 1689.674],
+ [700, 0.4, 0.1, 0.2728, 91.716],
+ [600, 0.4, 0.01, 0.27518, 201.72],
+ [5000, 2, 0.001, 0.26014, 717.804],
+ [1, 1, 0.0001, 0.38674, 961.776],
+ [10, 0.4, 0.1, 0.47818, 57.24],
+ [400, 0.1, 0.01, 0.27854, 192.874],
+ [200, 0.4, 0.01, 0.29728, 189.836],
+ [300, 0.8, 0.1, 0.28672, 87.502],
+ [2000, 0.4, 0.001, 0.2674, 275.852],
+ [0.1, 0.1, 0.0001, 0.4733, 1143.676],
+ [10, 0.2, 0.001, 0.43718, 428.824],
+ [2000, 0.3, 0.1, 0.2654, 98.448],
+ [600, 0.1, 0.01, 0.27172, 185.26],
+ [600, 0.3, 0.001, 0.27446, 394.724],
+ [2000, 1, 0.0001, 0.26225, 1308.055],
+ [300, 0.3, 0.1, 0.285675, 84.735],
+ [5, 0.2, 0.1, 0.4721, 66.178],
+ [7000, 0.2, 0.01, 0.2445, 313.41],
+ [4000, 0.1, 0.001, 0.24178, 596.902],
+ [1, 0.1, 0.0001, 0.47318, 1071.218],
+ [700, 0.4, 0.001, 0.2741, 347.246],
+ [700, 0.1, 0.01, 0.26968, 188.44],
+ [5, 0.6, 0.0001, 0.41224, 1064.182],
+ [400, 1, 0.01, 0.2844, 182.836],
+ [400, 0.1, 0.0001, 0.27872, 925.852],
+ [1, 0.6, 0.0001, 0.42206, 1050.308],
+ [0.1, 2, 0.01, 0.3443, 150.32],
+ [10000, 1, 0.0001, 0.26884, 237.932],
+ [800, 1, 0.1, 0.2881, 91.686],
+ [400, 1.5, 0.0001, 0.28484, 922.604],
+ [10, 0.9, 0.001, 0.37518, 439.848],
+ [10, 1.5, 0.1, 0.41416, 62.872],
+ [75, 0.8, 0.1, 0.36858, 73.028],
+ [700, 0.4, 0.01, 0.27374, 208.716],
+ [5000, 2, 0.0001, 0.26066, 1253.388],
+ [10, 0.6, 0.001, 0.3968, 424.838],
+ [1000, 0.9, 0.01, 0.2773, 179.5875],
+ [300, 2, 0.0001, 0.27404, 1621.78],
+ [6000, 1, 0.01, 0.2679, 188.96],
+ [4000, 0.3, 0.1, 0.25802, 116.096],
+ [2000, 0.8, 0.01, 0.27248, 156.818],
+ [700, 0.3, 0.0001, 0.272225, 802.165],
+ [0.1, 0.2, 0.1, 0.48458, 65.068],
+ [800, 0.8, 0.0001, 0.27668, 814.708],
+ [700, 0.3, 0.001, 0.27286, 328.432],
+ [4000, 0.8, 0.1, 0.26138, 119.046],
+ [800, 5, 0.01, 0.2838, 159.182],
+ [1000, 1.5, 0.001, 0.27984, 299.73],
+ [0.1, 2, 0.0001, 0.33334, 1079.002],
+ [500, 0.3, 0.0001, 0.27654, 935.272],
+ [4000, 0.4, 0.001, 0.24668, 628.582],
+ [7000, 0.3, 0.0001, 0.24534, 931.676],
+ [300, 0.4, 0.0001, 0.2868, 845.254],
+ [7000, 0.5, 0.001, 0.24786, 665.792],
+ [200, 0.5, 0.1, 0.296525, 83.8825],
+ [10000, 0.9, 0.001, 0.2514, 751.582],
+ [200, 0.2, 0.0001, 0.29764, 846.364],
+ [75, 0.5, 0.001, 0.33816, 419.206],
+ [10000, 1, 0.01, 0.26866, 164.224],
+ [7000, 2, 0.0001, 0.26488, 1284.018],
+ [75, 0.8, 0.0001, 0.32326, 816.718],
+ [25, 0.6, 0.1, 0.43602, 66.078],
+ [50, 0.9, 0.1, 0.39444, 73.242],
+ [6000, 0.1, 0.1, 0.25468, 114.67],
+ [10000, 0.8, 0.001, 0.25038, 660.802],
+ [600, 2, 0.01, 0.283, 180.698],
+ [300, 1.5, 0.01, 0.28718, 183.712],
+ [800, 3, 0.001, 0.26884, 683.236],
+ [25, 0.5, 0.1, 0.44598, 65.5],
+ [100, 0.2, 0.001, 0.34058, 463.52],
+ [1000, 0.8, 0.001, 0.27612, 296.306],
+ [75, 0.4, 0.1, 0.39524, 78.414],
+ [6000, 2, 0.0001, 0.26566, 1269.668],
+ [200, 0.9, 0.1, 0.29526, 86.62],
+ [3000, 0.8, 0.001, 0.25204, 679.502],
+ [700, 3, 0.0001, 0.2658, 1701.59],
+ [1, 2, 0.1, 0.36774, 74.084],
+ [0.1, 1.5, 0.01, 0.3562, 170.108],
+ [5000, 0.9, 0.1, 0.26558, 128.818],
+ [600, 0.9, 0.01, 0.28034, 191.452],
+ [800, 0.3, 0.01, 0.27124, 188.02],
+ [200, 5, 0.1, 0.28716, 78.842],
+ [400, 0.4, 0.001, 0.28116, 435.502],
+ [300, 1, 0.0001, 0.28928, 828.222],
+ [500, 3, 0.001, 0.2771, 600.59],
+ [2000, 3, 0.1, 0.27738, 112.184],
+ [800, 0.6, 0.001, 0.27528, 309.874],
+ [7000, 0.5, 0.0001, 0.24736, 1152.222],
+ [4000, 5, 0.01, 0.27848, 202.646],
+ [1000, 0.2, 0.01, 0.2685, 160.5],
+ [700, 0.2, 0.001, 0.27138, 345.794],
+ [800, 0.9, 0.001, 0.2781, 309.204],
+ [900, 3, 0.01, 0.2829, 157.484],
+ [1, 0.7, 0.001, 0.41394, 348.66],
+ [0.1, 1, 0.0001, 0.39142, 1045.278],
+ [700, 0.5, 0.1, 0.2731, 90.812],
+ [25, 0.3, 0.01, 0.4156, 209.686],
+ [200, 1, 0.0001, 0.29942, 773.04],
+ [4000, 1.5, 0.0001, 0.258, 1194.092],
+ [700, 0.7, 0.0001, 0.276125, 764.4775],
+ [3000, 0.2, 0.01, 0.26238, 161.98],
+ [3000, 1.5, 0.01, 0.27342, 171.822],
+ [800, 0.8, 0.01, 0.2767, 206.558],
+ [400, 2, 0.0001, 0.26658, 2024.066],
+ [700, 2, 0.01, 0.2825, 180.924],
+ [1000, 0.1, 0.0001, 0.266766666667, 549.47],
+ [10, 3, 0.01, 0.30906, 173.374],
+ [75, 0.2, 0.01, 0.3706, 198.368],
+ [10, 0.5, 0.0001, 0.41132, 1045.728],
+ [75, 1, 0.001, 0.33434, 388.386],
+ [600, 1.5, 0.0001, 0.26348, 2057.35],
+ [3000, 0.2, 0.0001, 0.24392, 1139.442],
+ [1, 0.8, 0.1, 0.4312, 68.044],
+ [3000, 0.2, 0.1, 0.26184, 99.994],
+ [1000000, 0.5, 0.0001, 0.5, 744.08],
+ [10, 0.5, 0.1, 0.48128, 55.422],
+ [5, 5, 0.01, 0.29864, 161.186],
+ [10, 1, 0.01, 0.3884, 169.096],
+ [75, 1.5, 0.0001, 0.30842, 747.038],
+ [400, 5, 0.1, 0.28514, 85.422],
+ [500, 0.4, 0.0001, 0.27766, 927.036],
+ [800, 1.5, 0.01, 0.28066, 192.67],
+ [75, 0.6, 0.1, 0.37946, 80.014],
+ [50, 2, 0.001, 0.3087, 345.298],
+ [5000, 0.3, 0.01, 0.26076, 158.292],
+ [25, 0.1, 0.01, 0.44188, 216.076],
+ [3000, 0.9, 0.0001, 0.25242, 1197.638],
+ [1000, 0.1, 0.001, 0.26702, 289.136],
+ [1, 0.6, 0.1, 0.45142, 67.71],
+ [25, 0.3, 0.001, 0.40472, 399.568],
+ [25, 1.5, 0.01, 0.33584, 181.14],
+ [50, 0.8, 0.1, 0.41962, 68.658],
+ [6000, 0.4, 0.0001, 0.24652, 1056.842],
+ [75, 1, 0.0001, 0.33392, 892.074],
+ [50, 0.1, 0.001, 0.4023, 442.506],
+ [5, 0.3, 0.1, 0.46282, 67.408],
+ [3000, 2, 0.0001, 0.26096, 1214.92],
+ [75, 0.3, 0.1, 0.4037, 76.695],
+ [5, 0.3, 0.001, 0.4349, 334.592],
+ [50, 0.7, 0.001, 0.34526, 394.276],
+ [300, 0.6, 0.01, 0.2872, 181.992],
+ [300, 5, 0.0001, 0.26854, 1504.19],
+ [3000, 0.5, 0.1, 0.26476, 111.6],
+ [600, 5, 0.0001, 0.2672, 1545.094],
+ [25, 5, 0.0001, 0.32426, 785.124],
+ [50, 0.6, 0.001, 0.3513, 415.802],
+ [900, 1, 0.1, 0.2869, 78.98],
+ [6000, 5, 0.0001, 0.26836, 1504.232],
+ [0.1, 0.4, 0.0001, 0.44742, 1212.744],
+ [1, 0.1, 0.01, 0.46678, 190.252],
+ [50, 5, 0.001, 0.28122, 525.002],
+ [600, 0.5, 0.1, 0.27402, 96.22],
+ [0.1, 0.6, 0.01, 0.42914, 168.89],
+ [10, 0.8, 0.01, 0.3865, 185.748],
+ [0.1, 5, 0.0001, 0.291, 1158.952],
+ [800, 0.1, 0.1, 0.26522, 91.154],
+ [300, 1.5, 0.1, 0.28748, 82.438],
+ [400, 3, 0.001, 0.28174, 491.63],
+ [400, 0.2, 0.0001, 0.27942, 926.686],
+ [1000, 5, 0.01, 0.28328, 162.104],
+ [50, 0.9, 0.0001, 0.33382, 890.334],
+ [1000000, 0.4, 0.0001, 0.5, 700.214],
+ [5, 0.1, 0.0001, 0.46014, 1152.888],
+ [5000, 1.5, 0.0001, 0.26604, 1212.062],
+ [4000, 3, 0.001, 0.26524, 813.44],
+ [800, 1.5, 0.1, 0.27958, 86.498],
+ [200, 0.5, 0.001, 0.29678, 353.592],
+ [5000, 0.4, 0.01, 0.26206, 162.022],
+ [50, 1.5, 0.01, 0.32012, 189.9],
+ [700, 1.5, 0.01, 0.2813, 196.356],
+ [6000, 5, 0.01, 0.2769, 249.764],
+ [200, 0.7, 0.001, 0.29606, 346.064],
+ [5000, 5, 0.0001, 0.26812, 1683.928],
+ [2000, 0.7, 0.1, 0.26908, 117.7],
+ [1, 3, 0.001, 0.31172, 349.082],
+ [3000, 0.5, 0.01, 0.26724, 160.404],
+ [0.1, 0.9, 0.001, 0.39486, 426.182],
+ [6000, 0.6, 0.001, 0.24924, 611.438],
+ [200, 0.7, 0.0001, 0.29554, 925.326],
+ [500, 2, 0.001, 0.2848, 366.786],
+ [6000, 2, 0.1, 0.27034, 156.278],
+ [1, 0.4, 0.01, 0.44532, 174.68],
+ [5000, 0.1, 0.01, 0.25844, 167.61],
+ [900, 0.9, 0.001, 0.27712, 302.426],
+ [10, 1, 0.0001, 0.38288, 992.33],
+ [2000, 2, 0.1, 0.27476, 104.136],
+ [500, 0.8, 0.01, 0.28112, 211.398],
+ [300, 0.5, 0.001, 0.28712, 367.25],
+ [300, 0.4, 0.01, 0.287, 194.978],
+ [800, 0.4, 0.1, 0.27028, 88.328],
+ [300, 0.3, 0.001, 0.28662, 376.584],
+ [50, 1.5, 0.001, 0.31738, 366.8],
+ [0.1, 3, 0.1, 0.35762, 62.382],
+ [100, 0.6, 0.1, 0.3382, 79.99],
+ [300, 0.2, 0.001, 0.28686, 386.25],
+ [500, 0.1, 0.1, 0.27172, 86.518],
+ [600, 0.6, 0.001, 0.2775, 397.966],
+ [100, 0.4, 0.01, 0.33208, 229.522],
+ [0.1, 0.4, 0.01, 0.4461, 171.486],
+ [500, 0.5, 0.001, 0.27848, 403.954],
+ [2000, 2, 0.0001, 0.26095, 1268.1725],
+ [5000, 1.5, 0.001, 0.25724, 777.294],
+ [300, 1.5, 0.001, 0.28822, 346.746],
+ [10000, 0.9, 0.01, 0.25704, 315.816],
+ [0.1, 0.6, 0.1, 0.45686, 58.824],
+ [100, 0.4, 0.001, 0.32884, 417.614],
+ [400, 0.5, 0.01, 0.28232, 195.978],
+ [7000, 0.3, 0.01, 0.2452, 324.976],
+ [2000, 1.5, 0.001, 0.25874, 769.75],
+ [75, 0.7, 0.0001, 0.32746, 820.95],
+ [7000, 0.4, 0.1, 0.25728, 131.03],
+ [3000, 1, 0.0001, 0.25684, 1031.304],
+ [25, 0.8, 0.1, 0.38342, 79.944],
+ [7000, 2, 0.1, 0.2716, 145.62],
+ [1000, 0.7, 0.0001, 0.259833333333, 1582.12666667],
+ [4000, 1, 0.01, 0.2686, 180.24],
+ [2000, 0.9, 0.01, 0.27362, 166.51],
+ [800, 1.5, 0.0001, 0.26236, 1820.576],
+ [100, 0.2, 0.01, 0.34562, 217.342],
+ [4000, 1, 0.0001, 0.25598, 859.122],
+ [4000, 0.2, 0.01, 0.26156, 151.65],
+ [300, 0.8, 0.001, 0.28742, 352.662],
+ [2000, 0.1, 0.01, 0.2629, 147.184],
+ [5, 0.6, 0.001, 0.40756, 358.234],
+ [25, 1.5, 0.0001, 0.35548, 969.826],
+ [5, 2, 0.01, 0.33614, 160.208],
+ [1000, 0.1, 0.1, 0.2649, 90.124],
+ [3000, 5, 0.01, 0.27922, 197.538],
+ [100, 2, 0.1, 0.30396, 68.108],
+ [1, 0.7, 0.0001, 0.41152, 1070.634],
+ [2000, 0.1, 0.001, 0.26284, 270.556],
+ [600, 0.2, 0.1, 0.26956, 85.388],
+ [100, 0.7, 0.001, 0.31746, 405.118],
+ [10000, 0.1, 0.001, 0.24136, 574.17],
+ [900, 0.3, 0.0001, 0.2702, 636.793333333],
+ [25, 3, 0.1, 0.335, 72.238],
+ [600, 5, 0.001, 0.26752, 853.444],
+ [75, 0.1, 0.01, 0.38056, 212.738],
+ [10000, 0.1, 0.01, 0.24206, 331.946],
+ [5000, 0.3, 0.0001, 0.2454, 1048.786],
+ [75, 1.5, 0.01, 0.3122, 180.414],
+ [600, 0.8, 0.0001, 0.27882, 854.386],
+ [3000, 0.5, 0.0001, 0.24836, 1191.178],
+ [3000, 2, 0.01, 0.27458, 167.476],
+ [300, 0.3, 0.01, 0.2867, 192.884],
+ [25, 0.1, 0.0001, 0.42854, 952.794],
+ [400, 0.7, 0.01, 0.28258, 190.108],
+ [50, 5, 0.1, 0.33886, 68.894],
+ [75, 5, 0.001, 0.32194, 393.94],
+ [100, 5, 0.01, 0.28724, 182.942],
+ [1000000, 0.1, 0.1, 0.5, 70.518],
+ [10, 0.7, 0.001, 0.3897, 410.686],
+ [800, 1, 0.01, 0.2761, 178.224],
+ [300, 0.4, 0.001, 0.287, 371.626],
+ [2000, 3, 0.0001, 0.2646, 1409.9425],
+ [200, 3, 0.1, 0.28714, 83.776],
+ [10, 0.3, 0.0001, 0.42976, 1064.32],
+ [5, 0.3, 0.01, 0.4471, 186.58],
+ [0.1, 0.3, 0.01, 0.45306, 172.472],
+ [700, 0.9, 0.01, 0.27872, 190.404],
+ [25, 0.4, 0.001, 0.39424, 435.566],
+ [5, 0.2, 0.01, 0.45266, 173.456],
+ [75, 0.2, 0.1, 0.42646, 80.556],
+ [2000, 1, 0.001, 0.27244, 205.764],
+ [700, 1.5, 0.1, 0.28095, 93.5225],
+ [75, 0.7, 0.01, 0.33256, 206.332],
+ [2000, 0.6, 0.1, 0.26872, 109.012],
+ [0.1, 0.6, 0.001, 0.4213, 373.228],
+ [10, 0.8, 0.1, 0.48062, 50.652],
+ [300, 0.2, 0.1, 0.28524, 97.464],
+ [4000, 0.7, 0.01, 0.2663, 162.276],
+ [1000, 0.6, 0.0001, 0.265433333333, 1178.76333333],
+ [400, 0.3, 0.0001, 0.27994, 904.98],
+ [5000, 1, 0.1, 0.27236, 106.98],
+ [5000, 5, 0.01, 0.27662, 236.194],
+ [1, 0.2, 0.1, 0.47676, 61.724],
+ [900, 0.8, 0.001, 0.27668, 338.366],
+ [1000000, 0.1, 0.0001, 0.5, 152.856],
+ [800, 0.2, 0.001, 0.27018, 304.842],
+ [1000, 0.7, 0.001, 0.2752, 323.488],
+ [6000, 2, 0.001, 0.26032, 724.04],
+ [1000, 0.3, 0.01, 0.26954, 168.854],
+ [1000000, 5, 0.0001, 0.40118, 564.886],
+ [100, 0.6, 0.0001, 0.31892, 842.156],
+ [3000, 5, 0.001, 0.26808, 998.786],
+ [200, 0.5, 0.01, 0.2968, 202.116],
+ [100, 3, 0.1, 0.29562, 81.284],
+ [500, 0.8, 0.001, 0.28076, 405.124],
+ [5, 0.8, 0.1, 0.43854, 61.566],
+ [6000, 0.4, 0.001, 0.2463, 584.552],
+ [400, 0.9, 0.0001, 0.28352, 911.014],
+ [1000, 3, 0.001, 0.2651, 838.328],
+ [700, 3, 0.01, 0.28444, 163.718],
+ [200, 0.5, 0.0001, 0.29676, 840.092],
+ [200, 0.4, 0.001, 0.29712, 363.562],
+ [400, 0.1, 0.1, 0.277275, 90.1575],
+ [5, 1, 0.0001, 0.38436, 927.208],
+ [4000, 0.5, 0.1, 0.2599, 117.092],
+ [900, 0.7, 0.01, 0.27506, 186.6],
+ [900, 1, 0.001, 0.2774, 338.136],
+ [0.1, 1, 0.001, 0.38732, 369.28],
+ [500, 0.5, 0.0001, 0.2784, 926.586],
+ [7000, 0.2, 0.001, 0.24314, 557.366],
+ [2000, 3, 0.01, 0.28084, 167.73],
+ [10, 1.5, 0.001, 0.34178, 446.116],
+ [1000000, 0.4, 0.001, 0.5, 284.836],
+ [10, 1, 0.1, 0.45074, 44.92],
+ [1, 2, 0.0001, 0.32726, 991.374],
+ [100, 0.9, 0.001, 0.31214, 398.49],
+ [0.1, 0.7, 0.1, 0.45412, 58.87],
+ [6000, 1.5, 0.01, 0.2663, 232.714],
+ [4000, 0.9, 0.1, 0.26394, 123.406],
+ [800, 0.3, 0.001, 0.27176, 314.438],
+ [7000, 0.4, 0.01, 0.2463, 334.802],
+ [4000, 0.5, 0.0001, 0.24796, 1206.42],
+ [800, 0.8, 0.001, 0.2773, 307.122],
+ [500, 0.1, 0.01, 0.2742, 193.062],
+ [600, 0.6, 0.0001, 0.2768, 867.236],
+ [700, 0.9, 0.001, 0.27934, 339.444],
+ [900, 5, 0.001, 0.27016, 673.782],
+ [25, 0.9, 0.1, 0.4167, 68.048],
+ [3000, 1, 0.001, 0.26962, 299.682],
+ [5, 0.9, 0.1, 0.40614, 65.708],
+ [1000, 0.8, 0.01, 0.27578, 189.112],
+ [500, 1.5, 0.1, 0.28482, 90.764],
+ [50, 0.7, 0.1, 0.43114, 61.586],
+ [10, 0.9, 0.0001, 0.3779, 1261.55],
+ [500, 0.4, 0.01, 0.2779, 196.13],
+ [2000, 0.6, 0.001, 0.26994, 306.254],
+ [75, 0.1, 0.0001, 0.37276, 933.128],
+ [200, 1.5, 0.1, 0.29206, 83.252],
+ [800, 5, 0.001, 0.26794, 754.956],
+ [600, 0.4, 0.001, 0.27542, 394.648],
+ [2000, 0.4, 0.1, 0.26664, 102.026],
+ [1000000, 0.2, 0.01, 0.5, 183.916],
+ [5, 0.7, 0.1, 0.4343, 60.78],
+ [7000, 1, 0.01, 0.2702, 158.588],
+ [5000, 0.8, 0.1, 0.2635, 126.588],
+ [7000, 0.8, 0.0001, 0.25264, 1118.04],
+ [75, 5, 0.1, 0.30028, 78.936],
+ [500, 0.4, 0.1, 0.27672, 90.402],
+ [0.1, 3, 0.01, 0.31722, 183.04],
+ [700, 0.6, 0.1, 0.27284, 87.81],
+ [5000, 0.5, 0.0001, 0.24796, 1184.634],
+ [5000, 0.6, 0.1, 0.26366, 126.656],
+ [500, 0.7, 0.001, 0.27976, 385.09],
+ [1000000, 1.5, 0.01, 0.49762, 124.654],
+ [1, 3, 0.01, 0.31604, 190.664],
+ [700, 0.5, 0.01, 0.27502, 193.652],
+ [1, 0.5, 0.001, 0.43322, 351.976],
+ [5, 0.7, 0.0001, 0.40232, 1035.59],
+ [500, 0.2, 0.001, 0.27568, 391.312],
+ [25, 1, 0.001, 0.36832, 361.092],
+ [400, 2, 0.1, 0.28216, 81.332],
+ [5, 3, 0.0001, 0.30488, 1159.526],
+ [0.1, 0.5, 0.1, 0.4743, 58.502],
+ [6000, 0.3, 0.001, 0.24502, 567.554],
+ [1, 0.5, 0.0001, 0.43388, 1087.284],
+ [900, 0.1, 0.001, 0.26776, 298.986],
+ [400, 0.6, 0.0001, 0.28224, 933.726],
+ [900, 0.1, 0.1, 0.265, 97.4975],
+ [50, 5, 0.0001, 0.27566, 1293.048],
+ [5000, 3, 0.0001, 0.26562, 1611.418],
+ [10000, 1, 0.1, 0.28032, 109.504],
+ [10, 0.9, 0.1, 0.46826, 61.228],
+ [700, 0.5, 0.001, 0.27522, 327.876],
+ [6000, 5, 0.001, 0.2679, 842.408],
+ [700, 2, 0.1, 0.2816, 84.222],
+ [300, 0.5, 0.0001, 0.28678, 869.392],
+ [6000, 0.4, 0.1, 0.25764, 123.252],
+ [75, 3, 0.1, 0.29808, 84.79],
+ [10, 0.8, 0.0001, 0.38592, 1187.246],
+ [100, 1, 0.0001, 0.32384, 834.042],
+ [400, 0.6, 0.001, 0.28266, 385.716],
+ [10, 0.5, 0.01, 0.41406, 179.11],
+ [0.1, 0.8, 0.1, 0.44602, 60.576],
+ [800, 2, 0.01, 0.28262, 149.586],
+ [10000, 0.9, 0.0001, 0.28978, 1136.934],
+ [5, 1, 0.01, 0.39272, 155.548],
+ [25, 0.7, 0.01, 0.3753, 227.884],
+ [7000, 0.2, 0.1, 0.25526, 127.32],
+ [400, 0.1, 0.001, 0.27908, 389.69],
+ [100, 2, 0.0001, 0.2855, 1342.104],
+ [200, 0.9, 0.001, 0.29516, 322.786],
+ [900, 5, 0.0001, 0.2681, 1377.79333333],
+ [200, 1, 0.01, 0.30064, 182.916],
+ [50, 0.8, 0.01, 0.34418, 215.614],
+ [300, 0.5, 0.01, 0.2875, 191.246],
+ [1, 0.7, 0.01, 0.41388, 178.784],
+ [7000, 5, 0.0001, 0.26844, 1511.754],
+ [900, 0.9, 0.0001, 0.255766666667, 2085.64666667],
+ [75, 3, 0.001, 0.32722, 428.408],
+ [10000, 3, 0.01, 0.26828, 313.056],
+ [3000, 2, 0.1, 0.27448, 114.978],
+ [1, 0.8, 0.01, 0.4089, 177.198],
+ [3000, 0.7, 0.1, 0.2679, 118.854],
+ [4000, 0.3, 0.01, 0.26284, 156.764],
+ [25, 2, 0.0001, 0.3441, 909.126],
+ [400, 1.5, 0.001, 0.2853, 379.38],
+ [1, 1.5, 0.001, 0.35434, 362.86],
+ [800, 0.7, 0.01, 0.27586, 187.5],
+ [500, 0.7, 0.0001, 0.28006, 943.238],
+ [6000, 0.7, 0.0001, 0.25076, 1110.904],
+ [500, 0.9, 0.0001, 0.28142, 941.56],
+ [800, 0.2, 0.0001, 0.2701, 651.41],
+ [10, 1, 0.001, 0.37668, 386.812],
+ [5000, 0.5, 0.001, 0.2482, 584.28],
+ [10000, 0.4, 0.1, 0.25832, 138.412],
+ [100, 0.3, 0.001, 0.3333, 422.244],
+ [10000, 0.6, 0.0001, 0.254, 1078.206],
+ [4000, 0.4, 0.1, 0.26012, 116.442],
+ [6000, 0.9, 0.01, 0.2623, 221.634],
+ [5, 0.9, 0.0001, 0.38618, 1200.626],
+ [4000, 0.6, 0.01, 0.26582, 165.468],
+ [5000, 1, 0.01, 0.26698, 198.29],
+ [0.1, 5, 0.1, 0.3441, 59.698],
+ [800, 0.1, 0.0001, 0.26838, 711.628],
+ [900, 0.3, 0.1, 0.26824, 98.986],
+ [5000, 0.1, 0.001, 0.24166, 558.132],
+ [100, 0.1, 0.1, 0.3972, 85.508],
+ [900, 0.4, 0.01, 0.27182, 184.53],
+ [700, 0.9, 0.0001, 0.277725, 810.8575],
+ [700, 3, 0.1, 0.28146, 88.434],
+ [2000, 1, 0.1, 0.2733, 85.972],
+ [10000, 5, 0.1, 0.28042, 167.404],
+ [5, 0.5, 0.001, 0.41648, 365.746],
+ [50, 1, 0.0001, 0.3471, 967.686],
+ [10000, 0.5, 0.0001, 0.2471, 1077.628],
+ [6000, 0.9, 0.001, 0.2517, 626.65],
+ [10, 2, 0.1, 0.45516, 45.912],
+ [300, 0.6, 0.1, 0.28728, 89.146],
+ [600, 0.7, 0.001, 0.27844, 369.498],
+ [900, 0.6, 0.1, 0.27334, 93.652],
+ [3000, 0.5, 0.001, 0.24868, 706.666],
+ [400, 0.8, 0.0001, 0.28288, 921.16],
+ [7000, 0.6, 0.001, 0.24936, 607.588],
+ [800, 0.2, 0.1, 0.26838, 86.472],
+ [10000, 0.8, 0.0001, 0.27032, 1147.492],
+ [3000, 0.3, 0.0001, 0.24584, 1286.306],
+ [200, 0.1, 0.01, 0.30184, 194.42],
+ [25, 0.2, 0.01, 0.43052, 204.764],
+ [5, 0.3, 0.0001, 0.44132, 1021.024],
+ [700, 0.6, 0.0001, 0.275225, 789.6675],
+ [300, 1, 0.1, 0.29412, 73.232],
+ [100, 3, 0.001, 0.28634, 493.786],
+ [75, 0.2, 0.0001, 0.3602, 862.904],
+ [1, 0.6, 0.001, 0.4225, 353.728],
+ [600, 0.1, 0.0001, 0.27116, 809.304],
+ [0.1, 2, 0.1, 0.39858, 52.04],
+ [400, 0.7, 0.001, 0.2828, 392.862],
+ [1000000, 2, 0.0001, 0.42602, 514.108],
+ [900, 0.2, 0.01, 0.26916, 172.61],
+ [5, 0.7, 0.01, 0.40932, 170.262],
+ [25, 2, 0.001, 0.39002, 269.67],
+ [400, 0.5, 0.1, 0.28174, 90.886],
+ [10, 0.2, 0.0001, 0.43962, 961.244],
+ [300, 0.7, 0.01, 0.28742, 185.994],
+ [1000, 0.2, 0.001, 0.26874, 325.168],
+ [4000, 0.3, 0.0001, 0.24542, 996.174],
+ [5000, 0.7, 0.1, 0.26352, 120.336],
+ [5000, 0.2, 0.001, 0.24324, 634.6],
+ [10, 3, 0.0001, 0.30214, 1093.328],
+ [7000, 1, 0.0001, 0.26272, 424.826],
+ [100, 0.7, 0.0001, 0.31596, 864.23],
+ [0.1, 0.7, 0.01, 0.41802, 168.444],
+ [10, 0.9, 0.01, 0.37986, 178.462],
+ [1000000, 1.5, 0.001, 0.48788, 387.352],
+ [75, 2, 0.1, 0.31056, 72.662],
+ [6000, 3, 0.01, 0.27282, 258.288],
+ [0.1, 0.7, 0.0001, 0.41688, 1152.06],
+ [10, 0.6, 0.1, 0.47892, 55.994],
+ [100, 0.1, 0.0001, 0.34888, 923.942],
+ [4000, 0.6, 0.0001, 0.24914, 1113.7],
+ [200, 0.8, 0.1, 0.2959, 84.91],
+ [50, 3, 0.001, 0.2926, 478.132],
+ [4000, 0.8, 0.001, 0.25136, 645.918],
+ [0.1, 0.1, 0.001, 0.46842, 401.66],
+ [900, 0.2, 0.0001, 0.269333333333, 628.16],
+ [0.1, 0.5, 0.01, 0.4332, 170.198],
+ [4000, 0.9, 0.001, 0.2523, 739.958],
+ [5000, 1.5, 0.01, 0.26838, 225.152],
+ [400, 2, 0.001, 0.28562, 353.47],
+ [75, 3, 0.01, 0.298, 180.122],
+ [900, 0.6, 0.001, 0.27466, 310.13],
+ [50, 0.3, 0.0001, 0.3751, 989.71],
+ [1000000, 0.8, 0.001, 0.5, 293.656],
+ [100, 2, 0.001, 0.29794, 387.326],
+ [300, 5, 0.01, 0.28578, 148.172],
+ [7000, 0.5, 0.01, 0.24834, 340.102],
+ [25, 0.1, 0.001, 0.42824, 441.102],
+ [800, 0.4, 0.001, 0.27274, 300.816],
+ [10000, 0.1, 0.1, 0.2529, 119.038],
+ [1000, 0.2, 0.0001, 0.268366666667, 700.76],
+ [75, 0.4, 0.001, 0.34458, 421.642],
+ [1000, 2, 0.1, 0.2788, 91.44],
+ [1000000, 3, 0.0001, 0.41112, 474.012],
+ [10, 2, 0.001, 0.32652, 418.05],
+ [100, 0.5, 0.01, 0.32816, 207.79],
+ [700, 0.6, 0.01, 0.27578, 190.46],
+ [200, 0.4, 0.0001, 0.29706, 844.122],
+ [50, 3, 0.1, 0.34192, 71.846],
+ [200, 3, 0.001, 0.28944, 331.724],
+ [1000000, 0.3, 0.01, 0.5, 185.472],
+ [5000, 0.6, 0.0001, 0.24886, 1109.19],
+ [75, 5, 0.01, 0.28944, 174.842],
+ [10000, 0.6, 0.001, 0.24862, 655.32],
+ [100, 1, 0.1, 0.32568, 69.28],
+ [4000, 0.3, 0.001, 0.2452, 607.508],
+ [1000000, 0.3, 0.001, 0.5, 286.296],
+ [5, 0.5, 0.01, 0.4255, 173.882],
+ [75, 0.9, 0.01, 0.32616, 201.38],
+ [1000, 1.5, 0.0001, 0.2583, 1784.29666667],
+ [1, 0.9, 0.1, 0.42466, 67.466],
+ [300, 1, 0.001, 0.2898, 367.256],
+ [400, 0.4, 0.01, 0.2818, 196.702],
+ [50, 0.2, 0.0001, 0.38622, 1070.496],
+ [700, 3, 0.001, 0.2686, 729.396],
+ [300, 0.7, 0.1, 0.28576, 86.99],
+ [1000000, 1.5, 0.0001, 0.49076, 617.896],
+ [200, 0.9, 0.0001, 0.29428, 814.972],
+ [50, 0.4, 0.01, 0.37258, 220.142],
+ [1000, 0.7, 0.1, 0.27324, 91.482],
+ [3000, 0.6, 0.001, 0.24968, 652.682],
+ [1000, 1, 0.0001, 0.276066666667, 592.426666667],
+ [900, 5, 0.1, 0.28214, 98.342],
+ [3000, 0.7, 0.0001, 0.25082, 1230.43],
+ [7000, 5, 0.1, 0.27998, 154.486],
+ [200, 0.4, 0.1, 0.2941, 91.948],
+ [2000, 0.6, 0.0001, 0.2498, 1601.1825],
+ [50, 2, 0.01, 0.31218, 172.738],
+ [800, 0.3, 0.0001, 0.2715, 655.716],
+ [50, 0.9, 0.01, 0.34062, 210.148],
+ [100, 0.3, 0.01, 0.338, 219.066],
+ [400, 0.6, 0.01, 0.28214, 191.192],
+ [25, 0.7, 0.1, 0.43346, 63.682],
+ [1, 2, 0.01, 0.34318, 157.166],
+ [200, 0.2, 0.1, 0.3005, 89.43],
+ [2000, 1.5, 0.1, 0.27326, 119.99],
+ [2000, 0.3, 0.01, 0.26606, 154.594],
+ [600, 5, 0.01, 0.28478, 165.648],
+ [400, 0.4, 0.1, 0.28006, 88.578],
+ [300, 1, 0.01, 0.2899, 165.49],
+ [100, 3, 0.0001, 0.27662, 1497.498],
+ [200, 0.8, 0.001, 0.29538, 350.604],
+ [100, 0.9, 0.01, 0.3136, 196.822],
+ [10, 3, 0.1, 0.40006, 53.434],
+ [900, 0.7, 0.1, 0.27322, 94.258],
+ [100, 0.5, 0.001, 0.32406, 395.308],
+ [700, 0.1, 0.001, 0.2696, 336.958],
+ [7000, 1.5, 0.001, 0.25746, 807.698],
+ [10000, 3, 0.0001, 0.27012, 1494.092],
+ [1000000, 0.2, 0.001, 0.5, 286.64],
+ [400, 0.8, 0.001, 0.28344, 394.92],
+ [500, 5, 0.0001, 0.26744, 1643.9],
+ [1000000, 3, 0.001, 0.3913, 297.292],
+ [700, 1.5, 0.001, 0.28164, 338.14],
+ [1, 0.9, 0.001, 0.3948, 342.218],
+ [2000, 0.7, 0.0001, 0.251025, 1605.2675],
+ [700, 2, 0.001, 0.28264, 350.05],
+ [5, 5, 0.0001, 0.28868, 1156.032],
+ [200, 1.5, 0.001, 0.29284, 304.968],
+ [600, 0.9, 0.1, 0.27882, 84.56],
+ [6000, 0.7, 0.01, 0.2603, 229.446],
+ [100, 2, 0.01, 0.29868, 172.308],
+ [200, 0.2, 0.01, 0.29912, 189.606],
+ [600, 1, 0.0001, 0.27994, 807.098],
+ [50, 0.6, 0.1, 0.44374, 64.918],
+ [800, 3, 0.01, 0.28372, 162.956],
+ [75, 0.7, 0.1, 0.36432, 74.894],
+ [1000000, 0.2, 0.0001, 0.5, 630.72],
+ [10, 0.7, 0.0001, 0.3947, 1307.996],
+ [6000, 1, 0.1, 0.2681, 109.674],
+ [1, 3, 0.1, 0.34516, 75.282],
+ [3000, 0.1, 0.01, 0.2607, 161.56],
+ [500, 0.6, 0.001, 0.2793, 398.692],
+ [1, 1.5, 0.01, 0.36092, 178.768],
+ [800, 0.7, 0.1, 0.27452, 87.044],
+ [2000, 0.5, 0.1, 0.26826, 103.134],
+ [50, 0.2, 0.1, 0.46536, 63.04],
+ [6000, 0.4, 0.01, 0.25522, 237.966],
+ [700, 1, 0.01, 0.27778, 194.92],
+ [25, 0.8, 0.001, 0.36226, 358.518],
+ [300, 0.5, 0.1, 0.28722, 82.508],
+ [800, 0.5, 0.001, 0.27382, 306.05],
+ [5000, 0.9, 0.0001, 0.25806, 1134.084],
+ [1, 5, 0.001, 0.29414, 370.214],
+ [4000, 3, 0.1, 0.27302, 138.12],
+ [3000, 0.3, 0.1, 0.26374, 101.466],
+ [900, 0.4, 0.0001, 0.2715, 649.23],
+ [10, 0.7, 0.1, 0.47822, 51.954],
+ [1, 0.2, 0.0001, 0.46368, 1054.878],
+ [100, 0.7, 0.1, 0.33174, 81.142],
+ [1000000, 0.3, 0.0001, 0.5, 592.446],
+ [600, 0.4, 0.1, 0.2745, 85.688],
+ [50, 0.8, 0.001, 0.33976, 401.61],
+ [1000, 0.2, 0.1, 0.2659, 90.218],
+ [7000, 0.3, 0.001, 0.24494, 598.638],
+ [25, 0.6, 0.0001, 0.37596, 832.832],
+ [400, 0.6, 0.1, 0.28198, 89.902],
+ [2000, 2, 0.01, 0.27766, 153.04],
+ [200, 5, 0.0001, 0.27176, 1806.102],
+ [1000, 1, 0.001, 0.27696, 322.304],
+ [600, 0.7, 0.01, 0.2781, 190.826],
+ [75, 0.1, 0.1, 0.42998, 75.72],
+ [200, 0.6, 0.001, 0.29642, 352.51],
+ [5, 2, 0.001, 0.3311, 341.924],
+ [0.1, 0.7, 0.001, 0.4134, 397.274],
+ [3000, 0.6, 0.1, 0.2681, 115.338],
+ [6000, 0.5, 0.001, 0.24824, 582.726],
+ [0.1, 0.4, 0.1, 0.4713, 68.072],
+ [7000, 0.5, 0.1, 0.2587, 132.888],
+ [900, 1, 0.01, 0.276, 172.26],
+ [300, 0.7, 0.0001, 0.28646, 899.854],
+ [4000, 1.5, 0.01, 0.27096, 178.934],
+ [2000, 0.2, 0.001, 0.26458, 271.194],
+ [10, 0.6, 0.01, 0.4043, 175.698],
+ [500, 0.2, 0.01, 0.27588, 195.34],
+ [10, 0.3, 0.001, 0.42636, 451.284],
+ [600, 1.5, 0.1, 0.28152, 92.638],
+ [5, 0.4, 0.0001, 0.43032, 959.25],
+ [2000, 0.3, 0.001, 0.26586, 281.396],
+ [50, 2, 0.1, 0.36454, 59.206],
+ [3000, 0.4, 0.01, 0.26562, 164.406],
+ [400, 5, 0.01, 0.28514, 185.678],
+ [25, 2, 0.01, 0.32052, 168.026],
+ [75, 0.3, 0.001, 0.35208, 428.454],
+ [400, 0.9, 0.01, 0.283675, 204.375],
+ [700, 0.5, 0.0001, 0.274175, 762.55],
+ [5000, 0.1, 0.1, 0.25874, 114.512],
+ [50, 0.7, 0.0001, 0.34302, 917.336],
+ [400, 1, 0.0001, 0.28498, 797.834],
+ [400, 0.7, 0.0001, 0.28294, 924.09],
+ [5000, 5, 0.001, 0.2692, 821.262],
+ [25, 1, 0.01, 0.37202, 198.388],
+ [600, 5, 0.1, 0.28294, 91.746],
+ [800, 0.4, 0.0001, 0.27238, 697.346],
+ [2000, 0.1, 0.1, 0.2618, 95.55],
+ [1000000, 0.5, 0.001, 0.5, 285.638],
+ [500, 1.5, 0.01, 0.28332, 197.076],
+ [10000, 2, 0.01, 0.26388, 339.25],
+ [200, 3, 0.0001, 0.27642, 1491.856],
+ [5000, 0.4, 0.0001, 0.24666, 1026.348],
+ [500, 0.8, 0.1, 0.2789, 98.748],
+ [300, 0.6, 0.0001, 0.2867, 870.246],
+ [1, 0.3, 0.01, 0.44994, 180.756],
+ [10000, 5, 0.0001, 0.27162, 1332.466],
+ [7000, 1, 0.001, 0.26652, 266.122],
+ [700, 5, 0.001, 0.27086, 743.828],
+ [2000, 0.9, 0.001, 0.25782, 626.026],
+ [4000, 1, 0.001, 0.26218, 386.334],
+ [200, 2, 0.1, 0.28926, 76.676],
+ [6000, 5, 0.1, 0.27894, 167.252],
+ [500, 0.5, 0.01, 0.27828, 193.274],
+ [1, 2, 0.001, 0.33426, 326.492],
+ [25, 5, 0.001, 0.37526, 267.138],
+ [5000, 0.7, 0.0001, 0.25188, 1149.17],
+ [500, 0.1, 0.0001, 0.27402, 903.586],
+ [5, 0.4, 0.01, 0.43206, 181.42],
+ [700, 0.2, 0.01, 0.27104, 184.49],
+ [7000, 0.7, 0.0001, 0.26038, 1184.366],
+ [1000000, 0.7, 0.0001, 0.5, 937.806],
+ [10000, 0.7, 0.1, 0.26502, 138.042],
+ [50, 0.8, 0.0001, 0.33768, 920.122],
+ [1000000, 0.4, 0.1, 0.5, 94.278],
+ [200, 0.1, 0.1, 0.2998, 84.12],
+ [600, 1, 0.01, 0.27946, 191.316],
+ [300, 3, 0.1, 0.28736, 85.87],
+ [800, 0.6, 0.1, 0.27386, 91.616],
+ [75, 0.4, 0.01, 0.34828, 205.254],
+ [2000, 0.2, 0.0001, 0.24555, 1501.525],
+ [4000, 1.5, 0.001, 0.2577, 739.756],
+ [10000, 0.1, 0.0001, 0.24178, 884.736],
+ [25, 0.8, 0.01, 0.37338, 200.534],
+ [2000, 0.5, 0.001, 0.26852, 280.408],
+ [700, 0.4, 0.0001, 0.273425, 749.8925],
+ [5, 0.9, 0.001, 0.38372, 346.818],
+ [5000, 0.6, 0.001, 0.24936, 672.644],
+ [1000000, 1, 0.001, 0.5, 241.932],
+ [5000, 0.5, 0.1, 0.26154, 119.208],
+ [800, 0.3, 0.1, 0.27096, 95.728],
+ [50, 0.6, 0.0001, 0.3492, 972.898],
+ [75, 0.8, 0.01, 0.32876, 189.114],
+ [900, 1.5, 0.0001, 0.2597, 1681.10333333],
+ [10000, 0.3, 0.001, 0.24512, 549.748],
+ [500, 0.2, 0.1, 0.27362, 89.64],
+ [5, 0.6, 0.01, 0.4147, 187.312],
+ [6000, 0.8, 0.01, 0.25792, 256.226],
+ [50, 0.5, 0.01, 0.3633, 225.63],
+ [25, 1, 0.0001, 0.36562, 766.336],
+ [50, 1, 0.1, 0.42188, 61.778],
+ [4000, 5, 0.0001, 0.26902, 1287.17],
+ [4000, 2, 0.1, 0.26876, 126.878],
+ [10000, 0.7, 0.001, 0.24966, 648.432],
+ [1000000, 5, 0.01, 0.40782, 164.792],
+ [75, 0.6, 0.01, 0.33574, 201.58],
+ [200, 0.8, 0.0001, 0.2949, 811.482],
+ [3000, 0.8, 0.1, 0.2685, 113.084],
+ [500, 5, 0.01, 0.2854, 184.386],
+ [10, 5, 0.001, 0.29568, 448.936],
+ [900, 0.7, 0.0001, 0.274433333333, 752.956666667],
+ [3000, 3, 0.0001, 0.26484, 1455.642],
+ [800, 0.6, 0.01, 0.27466, 185.232],
+ [3000, 2, 0.001, 0.26052, 737.314],
+ [900, 0.3, 0.001, 0.27084, 308.846],
+ [600, 3, 0.1, 0.28134, 86.322],
+ [600, 2, 0.001, 0.28334, 370.98],
+ [5000, 0.3, 0.001, 0.24512, 654.348],
+ [10000, 0.3, 0.0001, 0.2453, 982.958],
+ [7000, 0.6, 0.0001, 0.24884, 1098.944],
+ [25, 0.5, 0.0001, 0.3842, 864.636],
+ [0.1, 0.1, 0.1, 0.49314, 57.258],
+ [400, 5, 0.001, 0.27158, 739.132],
+ [10000, 0.3, 0.1, 0.25726, 126.818],
+ [600, 0.6, 0.01, 0.27706, 185.206],
+ [4000, 0.9, 0.01, 0.26692, 186.9],
+ [600, 0.7, 0.0001, 0.27768, 863.284],
+ [200, 0.3, 0.001, 0.29762, 366.65],
+ [5, 1, 0.001, 0.3813, 319.426],
+ [1000000, 0.9, 0.0001, 0.5, 928.034],
+ [900, 0.8, 0.0001, 0.261333333333, 1660.26666667],
+ [1000, 5, 0.1, 0.28112, 97.866],
+ [5000, 0.8, 0.01, 0.26328, 208.986],
+ [3000, 0.1, 0.001, 0.25294, 437.07],
+ [75, 2, 0.001, 0.34188, 293.594],
+ [10000, 0.6, 0.1, 0.26244, 133.818],
+ [2000, 0.3, 0.0001, 0.247325, 1409.0175],
+ [75, 0.3, 0.01, 0.3593, 202.536],
+ [600, 0.1, 0.1, 0.26942, 87.906],
+ [600, 0.3, 0.01, 0.2745, 191.162],
+ [900, 2, 0.0001, 0.2615, 1549.42],
+ [10, 0.4, 0.001, 0.41518, 422.25],
+ [900, 2, 0.01, 0.28182, 159.49],
+ [1, 0.3, 0.1, 0.47044, 64.492],
+ [2000, 0.7, 0.01, 0.27174, 165.078],
+ [0.1, 0.6, 0.0001, 0.42714, 1138.712],
+ [10, 0.4, 0.0001, 0.41956, 995.474],
+ [900, 2, 0.001, 0.27722, 375.43],
+ [600, 2, 0.1, 0.28232, 84.838],
+ [100, 3, 0.01, 0.29276, 187.61],
+ [5000, 2, 0.1, 0.27428, 152.904],
+ [5, 1.5, 0.01, 0.35362, 169.582],
+ [100, 5, 0.1, 0.28992, 77.758],
+ [1000000, 0.9, 0.01, 0.5, 148.324],
+ [1000, 1, 0.01, 0.27556, 172.896],
+ [10000, 0.4, 0.001, 0.24626, 628.786],
+ [1, 0.5, 0.1, 0.4674, 63.166],
+ [10000, 1, 0.001, 0.26832, 185.464],
+ [800, 0.9, 0.01, 0.27794, 190.624],
+ [5, 0.5, 0.1, 0.4578, 63.876],
+ [6000, 0.9, 0.0001, 0.26106, 1177.182],
+ [10, 2, 0.01, 0.33096, 156.662],
+ [1000, 0.8, 0.1, 0.2737, 94.394],
+ [1, 0.8, 0.001, 0.40362, 366.294],
+ [5000, 0.2, 0.01, 0.26004, 159.232],
+ [1000000, 0.5, 0.1, 0.5, 100.744],
+ [3000, 0.8, 0.01, 0.27014, 171.012],
+ [5, 5, 0.001, 0.33466, 311.576],
+ [2000, 1, 0.01, 0.27308, 141.104],
+ [300, 0.3, 0.0001, 0.28654, 890.966],
+ [1000, 1.5, 0.01, 0.27984, 158.248],
+ [100, 5, 0.001, 0.27954, 500.72],
+ [7000, 1.5, 0.0001, 0.26804, 1142.934],
+ [600, 0.2, 0.001, 0.27292, 398.208],
+ [6000, 0.3, 0.0001, 0.24546, 1000.238],
+ [1000, 0.4, 0.1, 0.2715, 91.888],
+ [10000, 0.3, 0.01, 0.24476, 355.386],
+ [3000, 0.6, 0.01, 0.26838, 157.41],
+ [700, 0.7, 0.001, 0.27704, 358.83],
+ [500, 3, 0.1, 0.28218, 84.64],
+ [7000, 0.1, 0.0001, 0.24174, 883.884],
+ [200, 1.5, 0.0001, 0.2924, 759.53],
+ [7000, 0.7, 0.001, 0.24988, 690.188],
+ [800, 3, 0.1, 0.2811, 95.036],
+ [5000, 3, 0.1, 0.2751, 143.092],
+ [600, 0.8, 0.001, 0.27946, 373.264],
+ [5000, 5, 0.1, 0.27954, 156.902],
+ [25, 5, 0.01, 0.29264, 182.294],
+ [500, 0.5, 0.1, 0.27756, 84.498],
+ [800, 0.9, 0.1, 0.27502, 88.914],
+ [7000, 0.8, 0.001, 0.25026, 645.568],
+ [500, 1, 0.1, 0.28366, 59.952],
+ [7000, 0.3, 0.1, 0.2568, 122.306],
+ [6000, 1, 0.001, 0.26664, 267.806],
+ [1000, 0.3, 0.001, 0.26994, 293.288],
+ [2000, 0.5, 0.01, 0.269675, 161.0725],
+ [1, 0.8, 0.0001, 0.40154, 1007.07],
+ [700, 0.7, 0.01, 0.27678, 193.244],
+ [1000000, 1, 0.1, 0.5, 98.16],
+ [1000000, 1, 0.0001, 0.5, 551.464],
+ [300, 2, 0.01, 0.28792, 175.738],
+ [3000, 3, 0.01, 0.27734, 178.99],
+ [75, 2, 0.01, 0.30636, 163.706],
+ [4000, 2, 0.001, 0.2602, 752.348],
+ [900, 0.8, 0.01, 0.27606, 191.036],
+ [1000000, 2, 0.001, 0.43144, 288.618],
+ [2000, 3, 0.001, 0.26476, 809.162],
+ [50, 0.9, 0.001, 0.33526, 386.42],
+ [6000, 0.6, 0.1, 0.25934, 126.272],
+ [1000, 0.9, 0.0001, 0.254866666667, 1979.02666667],
+ [0.1, 3, 0.001, 0.31316, 463.938],
+ [500, 0.6, 0.0001, 0.27926, 930.308],
+ [7000, 0.6, 0.01, 0.24958, 345.62],
+ [6000, 2, 0.01, 0.26492, 294.578],
+ [75, 0.8, 0.001, 0.32452, 407.924],
+ [5, 2, 0.0001, 0.32516, 1014.256],
+ [1, 0.3, 0.0001, 0.45462, 1099.228],
+ [6000, 0.3, 0.1, 0.25578, 123.272],
+ [6000, 0.6, 0.0001, 0.25206, 1099.786],
+ [200, 2, 0.001, 0.29122, 310.672],
+ [300, 0.9, 0.01, 0.28798, 186.704],
+ [1000, 0.4, 0.0001, 0.271066666667, 619.126666667],
+ [7000, 0.9, 0.01, 0.25654, 312.386],
+ [100, 0.6, 0.01, 0.32282, 205.416],
+ [10000, 0.6, 0.01, 0.24862, 416.906],
+ [500, 0.4, 0.001, 0.27782, 438.408],
+ [1000000, 0.7, 0.1, 0.5, 104.332],
+ [900, 0.9, 0.01, 0.27708, 184.072],
+ [2000, 0.8, 0.0001, 0.251625, 1521.055],
+ [300, 0.2, 0.0001, 0.28658, 858.066],
+ [50, 0.3, 0.1, 0.46048, 61.21],
+ [0.1, 0.9, 0.01, 0.40586, 174.718],
+ [5, 0.4, 0.1, 0.46744, 62.868],
+ [2000, 5, 0.0001, 0.2704, 1425.32],
+ [1, 1, 0.1, 0.42018, 55.71],
+ [0.1, 2, 0.001, 0.33648, 341.982],
+ [10000, 0.2, 0.0001, 0.24312, 888.092],
+ [300, 0.9, 0.001, 0.28762, 344.586],
+ [200, 2, 0.0001, 0.285, 1334.518],
+ [700, 0.7, 0.1, 0.27408, 88.412],
+ [5000, 0.7, 0.01, 0.2633, 214.44],
+ [1, 1, 0.001, 0.38912, 300.174],
+ [6000, 0.5, 0.01, 0.25344, 281.848],
+ [3000, 0.9, 0.001, 0.25284, 712.316],
+ [5, 0.8, 0.0001, 0.39384, 1228.768],
+ [800, 0.9, 0.0001, 0.2689, 1342.402],
+ [75, 0.2, 0.001, 0.36206, 420.58],
+ [7000, 3, 0.01, 0.268, 326.912],
+ [500, 5, 0.1, 0.28222, 83.39],
+ [900, 1.5, 0.1, 0.27794, 91.02],
+ [700, 5, 0.0001, 0.267875, 1574.455],
+ [50, 3, 0.01, 0.3007, 168.09],
+ [600, 2, 0.0001, 0.26486, 1867.686],
+ [3000, 0.3, 0.001, 0.24994, 682.044],
+ [700, 0.3, 0.01, 0.2729, 196.1],
+ [1, 3, 0.0001, 0.3045, 1023.176],
+ [2000, 0.2, 0.01, 0.26422, 147.94],
+ [4000, 1.5, 0.1, 0.26866, 127.972],
+ [1, 0.3, 0.001, 0.4523, 360.758],
+ [1000000, 0.4, 0.01, 0.5, 182.784],
+ [100, 0.9, 0.0001, 0.31106, 856.688],
+ [7000, 0.9, 0.1, 0.265, 134.972],
+ [25, 5, 0.1, 0.33184, 74.256],
+ [5000, 1.5, 0.1, 0.268, 138.442],
+ [4000, 0.9, 0.0001, 0.25332, 1158.6],
+ [75, 0.6, 0.001, 0.33322, 418.482],
+ [900, 0.1, 0.01, 0.26734, 165.618],
+ [900, 0.1, 0.0001, 0.267533333333, 607.793333333],
+ [0.1, 0.3, 0.0001, 0.4572, 1165.654],
+ [400, 0.7, 0.1, 0.28154, 88.258],
+ [200, 0.3, 0.0001, 0.29718, 802.894],
+ [900, 0.2, 0.1, 0.26608, 88.58],
+ [5, 0.1, 0.001, 0.4545, 350.72],
+ [5000, 0.8, 0.0001, 0.25116, 1185.046],
+ [10000, 2, 0.1, 0.27674, 158.698],
+ [6000, 0.9, 0.1, 0.26248, 127.374],
+ [400, 1, 0.1, 0.28668, 70.84],
+ [2000, 5, 0.001, 0.26822, 842.558],
+ [400, 1, 0.001, 0.2852, 392.806],
+ [600, 0.5, 0.0001, 0.27612, 850.596],
+ [900, 1, 0.0001, 0.2764, 595.7],
+ [700, 1, 0.1, 0.29358, 79.154],
+ [700, 0.6, 0.001, 0.27606, 338.544],
+ [400, 0.9, 0.1, 0.284675, 88.84],
+ [25, 0.5, 0.001, 0.38518, 390.786],
+ [25, 0.3, 0.0001, 0.4047, 861.23],
+ [6000, 0.7, 0.1, 0.26084, 127.19],
+ [1000, 0.7, 0.01, 0.27508, 182.732],
+ [800, 0.5, 0.01, 0.27376, 193.84],
+ [4000, 0.1, 0.1, 0.25742, 107.622],
+ [5000, 0.4, 0.1, 0.26054, 121.938],
+ [50, 0.2, 0.001, 0.3883, 486.984],
+ [50, 1, 0.001, 0.34926, 386.432],
+ [200, 3, 0.01, 0.2898, 174.974],
+ [200, 0.6, 0.01, 0.29656, 193.826],
+ [50, 0.1, 0.1, 0.47024, 63.298],
+ [5, 1, 0.1, 0.43308, 62.966],
+ [4000, 2, 0.01, 0.2728, 180.1],
+ [10000, 0.2, 0.001, 0.2433, 526.762],
+ [200, 1.5, 0.01, 0.29302, 180.636],
+ [700, 0.8, 0.01, 0.27808, 194.188],
+ [1, 0.9, 0.01, 0.39834, 181.634],
+ [1000000, 0.6, 0.01, 0.5, 159.072],
+ [100, 0.8, 0.01, 0.31706, 207.868],
+ [5, 0.4, 0.001, 0.4254, 356.88],
+ [900, 0.9, 0.1, 0.27328, 88.198],
+ [900, 0.3, 0.01, 0.27014, 177.064],
+ [900, 0.4, 0.001, 0.27168, 302.282],
+ [200, 0.7, 0.1, 0.29492, 89.816],
+ [7000, 1, 0.1, 0.26866, 121.104],
+ [3000, 1.5, 0.0001, 0.25838, 1167.57],
+ [1000, 0.1, 0.01, 0.26648, 150.268],
+ [900, 0.4, 0.1, 0.27052, 88.124],
+ [800, 0.8, 0.1, 0.27528, 89.906],
+ [300, 5, 0.1, 0.28652, 81.852],
+ [1000, 3, 0.1, 0.281, 93.516],
+ [0.1, 0.5, 0.001, 0.43142, 395.082],
+ [25, 0.5, 0.01, 0.39414, 213.126],
+ [400, 3, 0.01, 0.28582, 189.45],
+ [0.1, 0.3, 0.001, 0.45076, 415.572],
+ [7000, 0.9, 0.0001, 0.2568, 1110.788],
+ [200, 0.3, 0.01, 0.29788, 196.306],
+ [300, 0.9, 0.0001, 0.28726, 871.56],
+ [10000, 0.2, 0.1, 0.2543, 123.528],
+ [0.1, 0.2, 0.001, 0.45972, 411.124],
+ [75, 2, 0.0001, 0.2923, 1278.086],
+ [10000, 0.5, 0.01, 0.24726, 383.876],
+ [3000, 0.4, 0.001, 0.2471, 701.77],
+ [10000, 0.9, 0.1, 0.26962, 138.332],
+ [5, 1.5, 0.1, 0.42806, 59.038],
+ [400, 0.3, 0.1, 0.27886, 85.946],
+ [800, 0.5, 0.1, 0.27346, 88.518],
+ [7000, 0.1, 0.001, 0.24146, 527.12],
+ [900, 0.6, 0.01, 0.27454, 195.314],
+ [5, 1.5, 0.0001, 0.34628, 1175.276],
+ [0.1, 0.2, 0.0001, 0.466, 1157.236],
+ [0.1, 1.5, 0.001, 0.3557, 418.608],
+ [300, 0.6, 0.001, 0.28734, 362.69],
+ [600, 0.5, 0.001, 0.2763, 369.874],
+ [2000, 5, 0.01, 0.281225, 173.8725],
+ [1, 0.2, 0.01, 0.45894, 177.294],
+ [25, 0.9, 0.001, 0.3548, 393.792],
+ [5, 0.8, 0.001, 0.39164, 355.038],
+ [100, 0.2, 0.1, 0.38874, 80.036],
+ [5000, 0.2, 0.0001, 0.2438, 944.946],
+ [25, 0.6, 0.001, 0.37614, 413.888],
+ [1000, 0.6, 0.01, 0.27346, 210.89],
+ [300, 2, 0.1, 0.2842, 79.252],
+ [25, 0.9, 0.0001, 0.35434, 830.464],
+ [4000, 0.7, 0.001, 0.25078, 716.664],
+ [600, 1.5, 0.001, 0.2825, 361.05],
+ [1000000, 0.1, 0.001, 0.5, 124.676],
+ [500, 3, 0.01, 0.2841, 191.144],
+ [5000, 1, 0.001, 0.26034, 365.852],
+ [700, 5, 0.01, 0.2848, 161.416],
+ [50, 0.4, 0.0001, 0.36498, 1013.074],
+ [1000000, 0.6, 0.0001, 0.5, 918.886],
+ [1000000, 0.7, 0.001, 0.5, 311.996],
+ [2000, 0.8, 0.1, 0.26996, 103.672],
+ [10000, 0.4, 0.01, 0.24666, 367.468],
+ [5, 2, 0.1, 0.38626, 53.478],
+ [0.1, 0.8, 0.001, 0.40396, 401.224],
+ [200, 1, 0.001, 0.3005, 320.878],
+ [100, 0.3, 0.0001, 0.33274, 891.804],
+ [10, 0.3, 0.01, 0.4367, 177.636],
+ [1, 1, 0.01, 0.39912, 157.822],
+ [4000, 0.8, 0.0001, 0.25138, 1119.464],
+ [5000, 0.3, 0.1, 0.2609, 136.39],
+ [10000, 0.2, 0.01, 0.24258, 334.754],
+ [300, 0.1, 0.1, 0.28304, 85.45],
+ [700, 1, 0.001, 0.27946, 365.704],
+ [100, 1, 0.001, 0.32406, 412.482],
+ [700, 0.2, 0.1, 0.26872, 89.436],
+ [50, 3, 0.0001, 0.28218, 1474.37],
+ [4000, 5, 0.1, 0.27622, 141.548],
+ [600, 3, 0.001, 0.27646, 618.504],
+ [500, 0.3, 0.01, 0.27672, 193.994],
+ [5, 0.2, 0.001, 0.44494, 362.062],
+ [800, 2, 0.001, 0.27798, 392.928],
+ [25, 0.3, 0.1, 0.4656, 61.826],
+ [6000, 0.1, 0.01, 0.25708, 160.788],
+ [900, 3, 0.001, 0.26852, 684.37],
+ [1000000, 0.1, 0.01, 0.5, 140.52],
+ [10000, 2, 0.001, 0.25988, 743.122],
+ [10000, 0.7, 0.01, 0.24962, 393.284],
+ [1000000, 5, 0.1, 0.2842, 103.308],
+ [6000, 0.8, 0.0001, 0.25508, 1145.044],
+ [1000, 0.4, 0.001, 0.27102, 297.974],
+ [100, 0.4, 0.1, 0.36014, 81.222],
+ [500, 0.6, 0.01, 0.2792, 199.318],
+ [75, 3, 0.0001, 0.28032, 1433.972],
+ [500, 3, 0.0001, 0.2662, 1750.338],
+ [400, 0.2, 0.1, 0.27794, 88.754],
+ [0.1, 0.9, 0.1, 0.44702, 61.09],
+ [0.1, 0.4, 0.001, 0.44028, 426.974],
+ [10, 0.1, 0.001, 0.44644, 427.25],
+ [75, 1, 0.1, 0.37068, 68.896],
+ [4000, 0.7, 0.0001, 0.25016, 1149.51],
+ [800, 0.5, 0.0001, 0.2735, 710.34],
+ [10, 0.1, 0.01, 0.4537, 186.542],
+ [2000, 0.1, 0.0001, 0.244475, 1474.9],
+ [10000, 3, 0.1, 0.27304, 172.976],
+ [1, 0.7, 0.1, 0.43742, 64.206],
+ [10, 1.5, 0.01, 0.3467, 180.542],
+ [25, 0.8, 0.0001, 0.36112, 825.902],
+ [700, 1, 0.0001, 0.278375, 719.02],
+ [10, 1.5, 0.0001, 0.33882, 1194.746],
+ [400, 0.8, 0.1, 0.28164, 84.476],
+ [25, 3, 0.0001, 0.33108, 1055.664],
+ [300, 0.8, 0.01, 0.28716, 187.23],
+ [75, 1.5, 0.001, 0.34712, 300.506],
+ [600, 0.7, 0.1, 0.27614, 93.624],
+ [25, 0.9, 0.01, 0.36598, 204.464],
+ [4000, 3, 0.0001, 0.2653, 1488.496],
+ [4000, 0.4, 0.0001, 0.24662, 1030.786],
+ [4000, 0.5, 0.01, 0.26518, 162.112],
+ [7000, 5, 0.01, 0.27704, 213.324],
+ [200, 0.6, 0.1, 0.29356, 85.848],
+ [3000, 0.7, 0.01, 0.26922, 165.608],
+ [500, 0.1, 0.001, 0.27444, 391.212],
+ [100, 0.2, 0.0001, 0.33904, 866.464],
+ [1000, 0.3, 0.1, 0.26862, 92.152],
+ [1000, 0.5, 0.0001, 0.2722, 640.96],
+ [700, 0.3, 0.1, 0.27088, 87.974],
+ [0.1, 0.8, 0.01, 0.41062, 183.924],
+ [800, 5, 0.0001, 0.26742, 1459.934],
+ [100, 0.7, 0.01, 0.3199, 205.46],
+ [0.1, 0.9, 0.0001, 0.39922, 1159.944],
+ [900, 2, 0.1, 0.28026, 90.612],
+ [10, 0.4, 0.01, 0.42598, 178.41],
+ [1000, 0.3, 0.0001, 0.2694, 596.753333333],
+ [50, 2, 0.0001, 0.29378, 1441.402],
+ [1000000, 0.6, 0.001, 0.5, 320.392],
+ [10000, 0.5, 0.001, 0.24746, 589.576],
+ [200, 0.3, 0.1, 0.3012, 83.166],
+ [3000, 1.5, 0.1, 0.27166, 131.988],
+ [10, 0.1, 0.0001, 0.44892, 967.918],
+ [3000, 0.1, 0.1, 0.26074, 101.744],
+ [6000, 0.1, 0.0001, 0.24188, 912.088],
+ [900, 0.2, 0.001, 0.26962, 293.434],
+ [75, 0.9, 0.001, 0.32148, 392.108],
+ [25, 3, 0.001, 0.38214, 267.306],
+ [100, 0.8, 0.1, 0.32448, 80.456],
+ [500, 1.5, 0.0001, 0.27062, 1863.484],
+ [1, 0.9, 0.0001, 0.39268, 1032.246],
+ [5000, 0.9, 0.01, 0.2654, 175.872],
+ [1, 5, 0.1, 0.32704, 72.588],
+ [75, 0.5, 0.01, 0.34164, 202.41],
+ [900, 0.5, 0.1, 0.27102, 93.722],
+ [0.1, 5, 0.001, 0.29542, 397.414],
+ [600, 0.6, 0.1, 0.27544, 90.444],
+ [100, 0.3, 0.1, 0.379125, 81.1075],
+ [25, 2, 0.1, 0.32646, 72.75],
+ [3000, 0.4, 0.1, 0.2636, 109.338],
+ [300, 3, 0.001, 0.28696, 348.452],
+ [500, 2, 0.1, 0.28196, 82.392],
+ [50, 0.1, 0.0001, 0.39982, 1006.968],
+ [5, 0.5, 0.0001, 0.42106, 962.666],
+ [10, 5, 0.01, 0.2939, 178.594],
+ [6000, 1.5, 0.0001, 0.26338, 1164.17],
+ [200, 1, 0.1, 0.29364, 80.334],
+ [400, 0.8, 0.01, 0.28336, 196.466],
+ [800, 0.1, 0.01, 0.26832, 188.172],
+ [4000, 0.8, 0.01, 0.26672, 176.216],
+ [25, 0.6, 0.01, 0.39018, 201.18],
+ [500, 1, 0.01, 0.28042, 182.33],
+ [5, 0.1, 0.01, 0.46082, 175.868]]
+
+ c_values = (0.10000000000000001,
+ 1.0,
+ 5.0,
+ 10.0,
+ 25.0,
+ 50.0,
+ 75.0,
+ 100.0,
+ 200.0,
+ 300.0,
+ 400.0,
+ 500.0,
+ 600.0,
+ 700.0,
+ 800.0,
+ 900.0,
+ 1000.0,
+ 2000.0,
+ 3000.0,
+ 4000.0,
+ 5000.0,
+ 6000.0,
+ 7000.0,
+ 10000.0,
+ 1000000.0)
+ alpha_values = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 1.5, 2, 3, 5)
+ epsilon_values = (0.1, 0.01, 0.001, 0.0001)
+
+ # Convert the array into an easily accesible tree
+ config_tree = dict()
+ for C_value in c_values:
+ config_tree[C_value] = dict()
+ for alpha_value in alpha_values:
+ config_tree[C_value][alpha_value] = dict()
+ for epsilon_value in epsilon_values:
+ config_tree[C_value][alpha_value][epsilon_value] = [np.inf, np.inf]
+
+ # Populate the tree
+ for config in svm_grid_configs:
+ config_tree[config[0]][config[1]][config[2]][0] = config[3]
+ config_tree[config[0]][config[1]][config[2]][1] = config[4]
+
+ print c, alpha, epsilon
+ if ret_time:
+ return config_tree[c_values[c]][alpha_values[alpha]][epsilon_values[epsilon]][1]
+ else:
+ return config_tree[c_values[c]][alpha_values[alpha]][epsilon_values[epsilon]][0]
diff --git a/benchmark_util.py b/HPOlib/benchmark_util.py
similarity index 91%
rename from benchmark_util.py
rename to HPOlib/benchmark_util.py
index 7b14a68d..946acaea 100644
--- a/benchmark_util.py
+++ b/HPOlib/benchmark_util.py
@@ -16,9 +16,13 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
+import logging
import sys
+logger = logging.getLogger("HPOlib.benchmark_util")
+
+
def parse_cli():
"""
Provide a generic command line interface for benchmarks. It will just parse
@@ -45,7 +49,8 @@ def parse_cli():
cli_args = sys.argv
found_params = False
skip = True
- for idx, arg in enumerate(cli_args):
+ iterator = enumerate(cli_args)
+ for idx, arg in iterator:
if skip:
skip = False
continue
@@ -67,15 +72,13 @@ def parse_cli():
"--params argument. Please change the order.")
elif arg[0] == "-" and arg[0:2] != "--" and found_params:
- if cli_args[idx+1][0] == "-":
- raise ValueError("Hyperparameter name is not allowed to have a "
- "leading minus %s" % cli_args[idx + 1])
parameters[cli_args[idx][1:]] = cli_args[idx+1]
elif arg[0] == "-" and arg[0:2] != "--" and not found_params:
raise ValueError("You either try to use arguments with only one lea"
"ding minus or try to specify a hyperparameter bef"
- "ore the --params argument.")
+ "ore the --params argument. %s" %
+ " ".join(cli_args))
elif not found_params:
raise ValueError("Illegal command line string, expected an argument"
@@ -85,4 +88,4 @@ def parse_cli():
raise ValueError("Illegal command line string, expected a hyperpara"
"meter starting with - but found %s" % (arg,))
- return args, parameters
\ No newline at end of file
+ return args, parameters
diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py
new file mode 100644
index 00000000..b89319f4
--- /dev/null
+++ b/HPOlib/check_before_start.py
@@ -0,0 +1,134 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import glob
+import imp
+import logging
+import os
+import subprocess
+import sys
+
+logger = logging.getLogger("HPOlib.check_before_start")
+
+
+def _check_runsolver():
+ # check whether runsolver is in path
+ process = subprocess.Popen("which runsolver", stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, shell=True,
+ executable="/bin/bash")
+ stdoutdata, _stderrdata = process.communicate()
+
+ if stdoutdata is not None and "runsolver" in stdoutdata:
+ pass
+ else:
+ raise Exception("Runsolver cannot not be found. "
+ "Are you sure that it's installed?\n"
+ "Your $PATH is: " + os.environ['PATH'])
+
+
+def _check_modules():
+ """Checks whether all dependencies are installed"""
+
+ try:
+ import numpy
+ if numpy.__version__ < "1.6.0":
+ logger.warning("WARNING: You are using a numpy %s < 1.6.0. This "
+ "might not work", numpy.__version__)
+ except:
+ raise ImportError("Numpy cannot be imported. Are you sure that it's installed?")
+
+ try:
+ import scipy
+ if scipy.__version__ < "0.12.0":
+ logger.warning("WARNING: You are using a scipy %s < 0.12.0. "
+ "This might not work", scipy.__version__)
+ except:
+ raise ImportError("Scipy cannot be imported. Are you sure that it's installed?")
+
+ try:
+ import theano
+ logger.debug("\tTheano: %s" % str(theano.__version__))
+ except ImportError:
+ logger.warning("Theano not found. You might need this to run some "
+ "more complex benchmarks!")
+
+ if 'cuda' not in os.environ['PATH']:
+ logger.warning("CUDA not in $PATH")
+
+
+def _check_config(experiment_dir):
+ # check whether config file exists
+ config_file = os.path.join(experiment_dir, "config.cfg")
+ if not os.path.exists(config_file):
+ logger.warn("There is no config.cfg in %s, all options need to be provided by CLI arguments" % experiment_dir)
+
+
+def check_optimizer(optimizer):
+ path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "optimizers", optimizer)
+ if os.path.isdir(path):
+ # User told us, e.g. "tpe"
+ # Optimizer is in our optimizer directory
+ # Now check how many versions are present
+ parser = glob.glob(os.path.join(path, '*_parser.py'))
+ if len(parser) > 1:
+ logger.critical("Sorry I don't know which optimizer to use: %s",
+ parser)
+ sys.exit(1)
+ version = parser[0][:-10]
+ elif len(glob.glob(path + '*_parser.py')) == 1:
+ parser = glob.glob(path + '*_parser.py')
+ version = parser[0][:-10]
+ elif len(glob.glob(path + '*_parser.py')) > 1:
+ # Note this is a different case
+ # User told us e.g. "tpe/hyperopt_august" but this was not specific enough
+ logger.critical("Sorry I don't know which optimizer to use: %s",
+ glob.glob(path + '*_parser.py'))
+ sys.exit(1)
+ else:
+ logger.critical("We cannot find: %s", path)
+ sys.exit(1)
+
+ # Now check the other stuff
+ if not os.path.exists(version + "Default.cfg"):
+ logger.critical("Sorry I cannot find the default config for your "
+ "optimizer: %sDefault.cfg", version)
+ sys.exit(1)
+ if not os.path.exists(version + ".py"):
+ logger.critical("Sorry I cannot find the script to call your "
+ "optimizer: %s.py", version)
+ sys.exit(1)
+
+ # Check the optimizer dependencies
+ optimizer_module = imp.load_source(version, version + ".py")
+ optimizer_module.check_dependencies()
+ return version
+
+
+# noinspection PyUnusedLocal
+def check_first(experiment_dir):
+ """ Do some checks before optimizer is loaded """
+ logger.info("Check config.cfg..",)
+ _check_config(experiment_dir)
+ logger.info("..passed")
+ logger.info("Check dependencies:")
+ logger.info("Runsolver..")
+ _check_runsolver()
+ logger.info("..passed")
+ logger.info("Check python_modules..")
+ _check_modules()
+ logger.info("..passed")
\ No newline at end of file
diff --git a/config_parser/__init__.py b/HPOlib/config_parser/__init__.py
similarity index 100%
rename from config_parser/__init__.py
rename to HPOlib/config_parser/__init__.py
diff --git a/HPOlib/config_parser/generalDefault.cfg b/HPOlib/config_parser/generalDefault.cfg
new file mode 100644
index 00000000..511fe2e9
--- /dev/null
+++ b/HPOlib/config_parser/generalDefault.cfg
@@ -0,0 +1,32 @@
+[HPOLIB]
+#Will be used for wrapping.py and SMAC
+run_instance = runsolver_wrapper.py
+use_own_time_measurement = True
+number_cv_folds = 1
+max_crash_per_cv = 3
+remove_target_algorithm_output = True
+number_of_concurrent_jobs = 1
+
+function_setup =
+function_teardown =
+
+#The mem-limit must be expressed in mega-bytes
+#The cpu-limit must be expressed in seconds (CPU time)
+#The time-limit must be expressed in seconds (wall clock time)
+runsolver_time_limit =
+total_time_limit =
+memory_limit =
+cpu_limit =
+
+# E.g. Information for Theano and cuda
+leading_runsolver_info =
+leading_algo_info =
+
+# If an algorithm can handle cv on it's own
+# and calls runsolver_wrapper instead of cv.py as a function
+handles_cv=0
+
+# Dummy options, which need to be filled out later
+number_of_jobs=
+function=
+result_on_terminate=
\ No newline at end of file
diff --git a/HPOlib/config_parser/parse.py b/HPOlib/config_parser/parse.py
new file mode 100644
index 00000000..ee1c6d6f
--- /dev/null
+++ b/HPOlib/config_parser/parse.py
@@ -0,0 +1,62 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import os
+import logging
+import ConfigParser
+
+
+logger = logging.getLogger("HPOlib.config_parser.parse")
+
+
+def parse_config(config_files, allow_no_value=True, optimizer_version="",
+ cli_values=None):
+ if type(config_files) is str:
+ if not os.path.isfile(config_files):
+ raise Exception('%s is not a valid file\n' % os.path.join(
+ os.getcwd(), config_files))
+ else:
+ for config_file in config_files:
+ if not os.path.isfile(config_file):
+ raise Exception('%s is not a valid file\n' % os.path.join(
+ os.getcwd(), config_file))
+
+ config = ConfigParser.SafeConfigParser(allow_no_value=allow_no_value)
+
+ config.read(config_files)
+ if cli_values is not None:
+ config.readfp(cli_values)
+ return config
+
+
+def check_config(config):
+ # --------------------------------------------------------------------------
+ # Check critical values
+ # --------------------------------------------------------------------------
+ if not config.has_option('HPOLIB', 'number_of_jobs') or \
+ config.get('HPOLIB', 'number_of_jobs') == '':
+ raise Exception('number_of_jobs not specified in .cfg')
+ if not config.has_option('HPOLIB', 'result_on_terminate') or \
+ config.get('HPOLIB', 'result_on_terminate') == '':
+ raise Exception('No result_on_terminate specified in .cfg')
+ if not config.has_option('HPOLIB', 'function') or \
+ config.get('HPOLIB', 'function') == '':
+ raise Exception('No function specified in .cfg')
+ if config.getint('HPOLIB', "number_cv_folds") < 1:
+ raise Exception("The number of crossvalidation folds must be at least one!")
+ return True
diff --git a/cv.py b/HPOlib/cv.py
similarity index 66%
rename from cv.py
rename to HPOlib/cv.py
index fd417141..688e79b1 100644
--- a/cv.py
+++ b/HPOlib/cv.py
@@ -16,57 +16,53 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-#!/usr/bin/env python
-
-import cPickle
+import logging
import os
import subprocess
import sys
import time
-from importlib import import_module
-
import numpy as np
-from config_parser.parse import parse_config
-from Experiment import Experiment
-from run_instance import run_instance
-from wrapping_util import format_traceback
+from HPOlib.Experiment import Experiment
+from HPOlib.wrapping_util import format_traceback, load_experiment_config_file
+
__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
__license__ = "3-clause BSD License"
__contact__ = "automl.org"
+logging.basicConfig(format='[%(levelname)s] [%(asctime)s:%(name)s] %('
+ 'message)s', datefmt='%H:%M:%S')
+hpolib_logger = logging.getLogger("HPOlib")
+hpolib_logger.setLevel(logging.INFO)
+logger = logging.getLogger("HPOlib.cv")
+
+
+# TODO: This should be in a util function sometime in the future
+# Is duplicated in runsolver_wrapper.py
+def get_optimizer():
+ return "_".join(os.getcwd().split("/")[-1].split("_")[0:-2])
+
-def loadExperimentFile():
- optimizer = os.getcwd().split("/")[-1].split("_")[0]
+def load_experiment_file():
+ optimizer = get_optimizer()
experiment = Experiment(".", optimizer)
return experiment
-def doCV(params, folds=10):
- print "Starting Cross validation"
+def do_cv(params, folds=10):
+ logger.info("Starting Cross validation")
sys.stdout.flush()
- optimizer = os.getcwd().split("/")[-1].split("_")[0]
+ optimizer = get_optimizer()
+ cfg = load_experiment_config_file()
- # Load the config file, this holds information about data, black box fn etc.
- try:
- cfg_filename = "config.cfg"
- cfg = parse_config(cfg_filename, allow_no_value=True)
- except:
- cfg_filename = "../config.cfg"
- cfg = parse_config(cfg_filename, allow_no_value=True)
-
- # Now evaluate $fold times
- if folds < 1: return np.NaN
-
# Store the results to hand them back to tpe and spearmint
results = []
try:
- print "###\n", params, "\n###\n"
- param_array = ["-" + str(param_name) + " " + str(params[param_name]) \
- for param_name in params]
+ logger.info("%s", params)
+ param_array = ["-" + str(param_name) + " " + str(params[param_name]) for param_name in params]
param_string = " ".join(param_array)
for fold in range(folds):
@@ -76,22 +72,19 @@ def doCV(params, folds=10):
# Cutofftime, cutofflength and seed can be safely ignored since they
# are read in runsolver_wrapper
runsolver_wrapper_script = "python " + \
- os.path.join(os.path.dirname(os.path.realpath(__file__)), \
- "runsolver_wrapper.py")
+ os.path.join(os.path.dirname(os.path.realpath(__file__)), "runsolver_wrapper.py")
cmd = "%s %d %s %d %d %d %s" % \
- (runsolver_wrapper_script, fold, optimizer, 0, 0, 0, \
- param_string)
- print "Calling command:\n", cmd
+ (runsolver_wrapper_script, fold, optimizer, 0, 0, 0, param_string)
+ logger.info("Calling command:\n%s", cmd)
process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE, shell=True, executable="/bin/bash")
- print
- print "----------------RUNNING RUNSOLVER_WRAPPER-------------------"
+ stderr=subprocess.PIPE, shell=True, executable="/bin/bash")
+ logger.info("--------------RUNNING RUNSOLVER_WRAPPER--------------")
stdoutdata, stderrdata = process.communicate()
if stdoutdata:
- print stdoutdata
+ logger.info(stdoutdata)
if stderrdata:
- print stderrdata
+ logger.error(stderrdata)
# Read the runsolver_wrapper output
lines = stdoutdata.split("\n")
@@ -104,16 +97,16 @@ def doCV(params, folds=10):
results.append(float(result_array[6].strip(",")))
break
- if result_string == None:
+ if result_string is None:
raise NotImplementedError("No result string available or result string doesn't contain SAT")
# If a specified number of runs crashed, quit the whole cross validation
# in order to save time.
- worst_possible = cfg.getfloat("DEFAULT", "result_on_terminate")
+ worst_possible = cfg.getfloat("HPOLIB", "result_on_terminate")
crashed_runs = np.nansum([0 if res != worst_possible else 1 for res in results])
- if crashed_runs >= cfg.getint("DEFAULT", "max_crash_per_cv"):
- print "Aborting CV because the number of crashes excceds the " \
- "configured max_crash_per_cv value"
+ if crashed_runs >= cfg.getint("HPOLIB", "max_crash_per_cv"):
+ logger.warning("Aborting CV because the number of crashes "
+ "exceeds the configured max_crash_per_cv value")
return worst_possible
# TODO: Error Handling
@@ -122,17 +115,17 @@ def doCV(params, folds=10):
mean = np.mean(results)
except Exception as e:
- print format_traceback(sys.exc_info())
- print "CV failed", sys.exc_info()[0], e
+ logger.error(format_traceback(sys.exc_info()))
+ logger.error("CV failed %s %s", sys.exc_info()[0], e)
# status = "CRASHED"
- status = "SAT"
+ # status = "SAT"
mean = np.NaN
# Do not return any kind of nan because this would break spearmint
if not np.isfinite(mean):
- mean = float(cfg.get("DEFAULT", "result_on_terminate"))
+ mean = float(cfg.get("HPOLIB", "result_on_terminate"))
- print "Finished CV"
+ logger.info("Finished CV")
return mean
@@ -152,8 +145,8 @@ def flatten_parameter_dict(params):
# This class enables us to distinguish tuples, which are parameters from
# tuples which contain different things...
class Parameter:
- def __init__(self, param):
- self.param = param
+ def __init__(self, pparam):
+ self.pparam = pparam
params_to_check = list()
params_to_check.append(params)
@@ -163,7 +156,6 @@ def __init__(self, param):
param = params_to_check.pop()
if type(param) in (list, tuple, np.ndarray):
- added_literal = False
for sub_param in param:
params_to_check.append(sub_param)
@@ -172,12 +164,12 @@ def __init__(self, param):
params_to_check.extend([Parameter(tmp_param) for tmp_param in zip(param.keys(), param.values())])
elif isinstance(param, Parameter):
- key = param.param[0]
- value = param.param[1]
+ key = param.pparam[0]
+ value = param.pparam[1]
if type(value) == dict:
params_to_check.append(value)
elif type(value) in (list, tuple, np.ndarray) and \
- all([type(v) not in (list, tuple, np.ndarray) for v in value]):
+ all([type(v) not in (list, tuple, np.ndarray) for v in value]):
# Spearmint special case, keep only the first element
new_dict[key] = value[0]
elif type(value) in (list, tuple, np.ndarray):
@@ -187,47 +179,56 @@ def __init__(self, param):
new_dict[key] = value
else:
- raise Exception("Invalid params, cannot be flattened.")
+ raise Exception("Invalid params, cannot be flattened: \n%s." % params)
params = new_dict
return params
-def main(job_id, params):
- # Forget job_id which comes from spearmint
+def main(*args, **kwargs):
+ logger.critical('args: %s kwargs: %s', str(args), str(kwargs))
+
+ params = None
+
+ if 'params' in kwargs.keys():
+ params = kwargs['params']
+ else:
+ for arg in args:
+ if type(arg) == dict:
+ params = arg
+
+ if params is None:
+ logger.critical("No parameter dict found in cv.py.\n"
+ "args: %s\n kwargs: %s", args, kwargs)
+ sys.exit(1)
# Load the experiment to do time-keeping
cv_starttime = time.time()
- experiment = loadExperimentFile()
+ experiment = load_experiment_file()
# experiment.next_iteration()
experiment.start_cv(cv_starttime)
del experiment
- # Load the config file, this holds information about data, black box fn etc.
- try:
- cfg_filename = "config.cfg"
- cfg = parse_config(cfg_filename, allow_no_value=True)
- except:
- cfg_filename = "../config.cfg"
- cfg = parse_config(cfg_filename, allow_no_value=True)
+ # cfg_filename = "config.cfg"
+ cfg = load_experiment_config_file()
# Load number of folds
- folds = cfg.getint('DEFAULT', 'numberCV')
+ folds = cfg.getint('HPOLIB', 'number_cv_folds')
params = flatten_parameter_dict(params)
- res = doCV(params, folds=folds)
- print "Result: ", res
+ res = do_cv(params, folds=folds)
+ logger.info("Result: %f", res)
# Load the experiment to do time-keeping
- experiment = loadExperimentFile()
+ experiment = load_experiment_file()
experiment.end_cv(time.time())
del experiment
return res
-def doForTPE(params):
- return main(42, params)
+#def doForTPE(params):
+# return main(42, params)
def read_params_from_command_line():
@@ -244,8 +245,8 @@ def read_params_from_command_line():
key = key[1:]
if value[0] != "'" or value[-1] != "'":
- raise ValueError("Expected parameter value %d to be inside single"
- "quotation marks." % (idx*2 + 1))
+ raise ValueError("Expected parameter value %s to be inside single"
+ "quotation marks." % value)
else:
value = value.strip("'")
params[key] = value
@@ -253,5 +254,5 @@ def read_params_from_command_line():
if __name__ == "__main__":
- params = read_params_from_command_line()
- print main(42, params)
+ cli_params = read_params_from_command_line()
+ main(params=cli_params)
diff --git a/data_util.py b/HPOlib/data_util.py
similarity index 82%
rename from data_util.py
rename to HPOlib/data_util.py
index d47c1b31..a979895f 100644
--- a/data_util.py
+++ b/HPOlib/data_util.py
@@ -18,20 +18,24 @@
import cPickle
from gzip import GzipFile as gfile
-import numpy as np
+import logging
import os
-import sys
+
+import numpy as np
__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
__contact__ = "automl.org"
+logger = logging.getLogger("HPOlib.data_util")
+
+
def load_file(filename, file_format, use_percentage):
if not os.path.exists(filename):
raise IOError("File %s not found", filename)
if file_format == "gfile":
- print "Loading file:", filename
+ logger.info("Loading file: %s", filename)
fh = gfile(filename, "rb")
data = cPickle.load(fh)
if use_percentage >= 100.:
@@ -40,11 +44,11 @@ def load_file(filename, file_format, use_percentage):
max_data = int(len(data) / 100. * use_percentage)
data = data[:max_data]
fh.close()
- print "Done loading file:", filename, "has", len(data), "datapoints"
- sys.stdout.flush()
+ logger.info("Done loading file: %s has %d datapoints", filename,
+ len(data))
elif file_format == "pickle":
- print "Loading file:", filename
+ logger.info("Loading file: %s", filename)
fh = open(filename, "r")
data = cPickle.load(fh)
if use_percentage >= 100.:
@@ -52,10 +56,11 @@ def load_file(filename, file_format, use_percentage):
else:
data = data[:len(data) / 100. * use_percentage]
fh.close()
- print "Done loading file:", filename, "has", len(data), "datapoints"
+ logger.info("Done loading file: %s has %d datapoints", filename,
+ len(data))
elif file_format == "numpy":
- print "Loading file:", filename
+ logger.info("Loading file: %s", filename)
fh = open(filename, "r")
data = np.load(fh)
if use_percentage >= 100.:
@@ -63,7 +68,8 @@ def load_file(filename, file_format, use_percentage):
else:
data = data[:len(data) / 100. * use_percentage]
fh.close()
- print "Done loading file:", filename, "has", len(data), "datapoints"
+ logger.info("Done loading file: %s has %d datapoints", filename,
+ len(data))
else:
raise ValueError("%s is an unknown training_data_format", file_format)
@@ -85,9 +91,6 @@ def custom_split(data, n_train, n_valid):
"data (%d)") % (n_train, n_valid, len(data))
train = data[0:n_train]
valid = data[n_train:]
- print type(train)
- print type(valid)
-
return train, valid
@@ -107,7 +110,8 @@ def prepare_cv_for_fold(data, fold, folds):
In case no data is handed over to the function, None is returned.
"""
- print "Fold", fold, "of ", folds, "folds"
+ logger.info("Splitting data:")
+ logger.info("Fold %s of %s folds", fold, folds)
# Create an array with the split points
if data is not None:
data_len = len(data)
@@ -125,7 +129,7 @@ def prepare_cv_for_fold(data, fold, folds):
data = np.array(data)
if isinstance(data, np.ndarray):
- cv_split_mask = np.empty((data_len), dtype = np.bool)
+ cv_split_mask = np.empty((data_len), dtype=np.bool)
for i in range(folds):
if i != fold:
cv_split_mask[splits[i]:splits[i+1]] = 1
@@ -133,7 +137,8 @@ def prepare_cv_for_fold(data, fold, folds):
cv_split_mask[splits[i]:splits[i+1]] = 0
train = data[cv_split_mask]
valid = data[~cv_split_mask]
- print data.shape, train.shape, valid.shape, train.itemsize
+ logger.info("%d, %d, %d, %d", data.shape, train.shape, valid.shape,
+ train.itemsize)
else:
train = []
valid = []
@@ -142,4 +147,4 @@ def prepare_cv_for_fold(data, fold, folds):
valid.append(datum)
else:
train.append(datum)
- return train, valid
\ No newline at end of file
+ return train, valid
diff --git a/format_converter/Readme.txt b/HPOlib/format_converter/Readme.txt
similarity index 100%
rename from format_converter/Readme.txt
rename to HPOlib/format_converter/Readme.txt
diff --git a/HPOlib/format_converter/__init__.py b/HPOlib/format_converter/__init__.py
new file mode 100644
index 00000000..163eead8
--- /dev/null
+++ b/HPOlib/format_converter/__init__.py
@@ -0,0 +1 @@
+__author__ = 'eggenspk'
diff --git a/HPOlib/format_converter/convert.py b/HPOlib/format_converter/convert.py
new file mode 100644
index 00000000..7233767e
--- /dev/null
+++ b/HPOlib/format_converter/convert.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+from argparse import ArgumentParser
+import os
+from string import upper
+import sys
+import tempfile
+
+import smac_to_spearmint
+import tpe_to_smac
+
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+def smac_to_spearmint_helper(space, save=""):
+ # print "Convert %s from SMAC to SPEARMINT" % space
+ return smac_to_spearmint.convert_smac_to_spearmint(space)
+
+
+def smac_to_tpe_helper(space, save=""):
+ print "This is not yet implemented"
+
+
+def spearmint_to_smac_helper(space, save=""):
+ print "This is not yet implemented"
+
+
+def spearmint_to_tpe_helper(space, save=""):
+ print "This is not yet implemented"
+
+
+def tpe_to_spearmint_helper(space, save=""):
+ try:
+ import hyperopt
+ except ImportError:
+ print "Cannot find hyperopt. To use this converter, modify $PYTHONPATH to contain a hyperopt installation"
+
+ # First convert to smac
+ tmp = tpe_to_smac.convert_tpe_to_smac_from_file(space)
+ handle, tmp_file_name = tempfile.mkstemp()
+ fh = open(tmp_file_name, 'w')
+ fh.write(tmp)
+ fh.close()
+
+ # From smac convert to spearmint
+ new_space = smac_to_spearmint.convert_smac_to_spearmint(tmp_file_name)
+
+ os.remove(tmp_file_name)
+ return new_space
+
+
+def tpe_to_smac_helper(space, save=""):
+ try:
+ import hyperopt
+ except ImportError:
+ print "Cannot find hyperopt. To use this converter, modify $PYTHONPATH to contain a hyperopt installation"
+ return tpe_to_smac.convert_tpe_to_smac_from_file(space)
+
+
+def main():
+ # python convert.py --from SMAC --to TPE -f space.any -s space.else
+ prog = "python convert.py"
+ description = "Automatically convert a searchspace from one format to another"
+
+ parser = ArgumentParser(description=description, prog=prog)
+
+ parser.add_argument("--from", dest="conv_from", choices=['SMAC', 'Smac', 'smac',
+ 'TPE', 'Tpe', 'tpe', 'hyperopt',
+ 'SPEARMINT', 'Spearmint', 'spearmint'],
+ default="", help="Convert from which format?", required=True)
+ parser.add_argument("--to", dest="conv_to", choices=['SMAC', 'Smac', 'smac',
+ 'TPE', 'Tpe', 'tpe', 'hyperopt',
+ 'SPEARMINT', 'Spearmint', 'spearmint'],
+ default="", help="Convert to which format?", required=True)
+ parser.add_argument("-f", "--file", dest="space",
+ default="", help="Where is the searchspace to be converted?", required=True)
+ parser.add_argument("-s", "--save", dest="save",
+ default="", help="Where to save the new searchspace?")
+
+ args, unknown = parser.parse_known_args()
+
+ space = os.path.abspath(args.space)
+ if not os.path.isfile(space):
+ print "%s is not a valid path" % space
+ sys.exit(1)
+
+ # Unifying strings
+ args.conv_to = upper(args.conv_to)
+ args.conv_from = upper(args.conv_from)
+ if args.conv_from == "HYPEROPT":
+ args.conv_from = "TPE"
+ if args.conv_to == "HYPEROPT":
+ args.conv_to == "TPE"
+
+ if args.conv_to == args.conv_from:
+ print "Converting from %s to %s makes no sense" % (args.conv_to, args.conv_from)
+
+ # This is like a switch statement
+ options = {'SMAC': {'SPEARMINT': smac_to_spearmint_helper,
+ 'TPE': smac_to_tpe_helper},
+ 'SPEARMINT': {'SMAC': spearmint_to_smac_helper,
+ 'TPE': spearmint_to_tpe_helper},
+ 'TPE': {'SPEARMINT': tpe_to_spearmint_helper,
+ 'SMAC': tpe_to_smac_helper}
+ }
+ new_space = options[args.conv_from][args.conv_to](space, args.save)
+ if args.save != "":
+ fh = open(args.save, 'w')
+ fh.write(new_space)
+ fh.close()
+ else:
+ print new_space
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/format_converter/SMACSpearmint.py b/HPOlib/format_converter/smac_to_spearmint.py
similarity index 84%
rename from format_converter/SMACSpearmint.py
rename to HPOlib/format_converter/smac_to_spearmint.py
index a71b051a..8c5e753e 100644
--- a/format_converter/SMACSpearmint.py
+++ b/HPOlib/format_converter/smac_to_spearmint.py
@@ -16,26 +16,25 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-#!/usr/bin/env python
"""
-This script reads a SMAC search space and prints out an spearmint searchspace
+This method reads a SMAC search space and returns a spearmint searchspace
Default value and conditions are lost
"""
import sys
import os
-path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../optimizers/")
-sys.path.append(path)
-from spearmint_april2013_mod.spearmint_pb2 import Experiment, PYTHON
+
+from spearmint_april2013_mod_spearmint_pb2 import Experiment, PYTHON
from google.protobuf import text_format
import numpy as np
-def convertSMACToSpearmint(filename):
- file = open(filename)
+
+def convert_smac_to_spearmint(filename):
+ fl_handle = open(filename)
params = []
- for line in file:
+ for line in fl_handle:
line = line.strip()
if line.strip().find('#') == 0 or line == "":
continue
@@ -97,11 +96,6 @@ def convertSMACToSpearmint(filename):
exp.language = PYTHON
exp.name = "cv"
exp.variable.extend(params)
- print text_format.MessageToString(exp)
-
-if __name__ == '__main__':
- if len(sys.argv) != 2:
- sys.stdout.write("Wrong number of Arguments\nUsage: python SMACSpearmint.py ")
- sys.exit(1)
- convertSMACToSpearmint(sys.argv[1])
+ return text_format.MessageToString(exp)
+
diff --git a/HPOlib/format_converter/spearmint_april2013_mod_spearmint_pb2.py b/HPOlib/format_converter/spearmint_april2013_mod_spearmint_pb2.py
new file mode 100644
index 00000000..f215e531
--- /dev/null
+++ b/HPOlib/format_converter/spearmint_april2013_mod_spearmint_pb2.py
@@ -0,0 +1,406 @@
+##
+# Copyright (C) 2012 Jasper Snoek, Hugo Larochelle and Ryan P. Adams
+#
+# This code is written for research and educational purposes only to
+# supplement the paper entitled
+# "Practical Bayesian Optimization of Machine Learning Algorithms"
+# by Snoek, Larochelle and Adams
+# Advances in Neural Information Processing Systems, 2012
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+##
+# Copyright (C) 2012 Jasper Snoek, Hugo Larochelle and Ryan P. Adams
+#
+# This code is written for research and educational purposes only to
+# supplement the paper entitled
+# "Practical Bayesian Optimization of Machine Learning Algorithms"
+# by Snoek, Larochelle and Adams
+# Advances in Neural Information Processing Systems, 2012
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+
+from google.protobuf import descriptor
+from google.protobuf import message
+from google.protobuf import reflection
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+
+
+DESCRIPTOR = descriptor.FileDescriptor(
+ name='spearmint.proto',
+ package='',
+ serialized_pb='\n\x0fspearmint.proto\"\xcc\x01\n\x03Job\x12\n\n\x02id\x18\x01 \x02(\x04\x12\x10\n\x08\x65xpt_dir\x18\x02 \x02(\t\x12\x0c\n\x04name\x18\x03 \x02(\t\x12\x1b\n\x08language\x18\x04 \x02(\x0e\x32\t.Language\x12\x0e\n\x06status\x18\x05 \x01(\t\x12\x19\n\x05param\x18\x06 \x03(\x0b\x32\n.Parameter\x12\x10\n\x08submit_t\x18\x07 \x01(\x04\x12\x0f\n\x07start_t\x18\x08 \x01(\x04\x12\r\n\x05\x65nd_t\x18\t \x01(\x04\x12\r\n\x05value\x18\n \x01(\x01\x12\x10\n\x08\x64uration\x18\x0b \x01(\x01\"L\n\tParameter\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\x0f\n\x07int_val\x18\x02 \x03(\x03\x12\x0f\n\x07str_val\x18\x03 \x03(\t\x12\x0f\n\x07\x64\x62l_val\x18\x04 \x03(\x01\"\x91\x02\n\nExperiment\x12\x1b\n\x08language\x18\x01 \x02(\x0e\x32\t.Language\x12\x0c\n\x04name\x18\x02 \x02(\t\x12+\n\x08variable\x18\x03 \x03(\x0b\x32\x19.Experiment.ParameterSpec\x1a\xaa\x01\n\rParameterSpec\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\x0c\n\x04size\x18\x02 \x02(\r\x12,\n\x04type\x18\x03 \x02(\x0e\x32\x1e.Experiment.ParameterSpec.Type\x12\x0f\n\x07options\x18\x04 \x03(\t\x12\x0b\n\x03min\x18\x05 \x01(\x01\x12\x0b\n\x03max\x18\x06 \x01(\x01\"$\n\x04Type\x12\x07\n\x03INT\x10\x01\x12\t\n\x05\x46LOAT\x10\x02\x12\x08\n\x04\x45NUM\x10\x03*6\n\x08Language\x12\n\n\x06MATLAB\x10\x01\x12\n\n\x06PYTHON\x10\x02\x12\t\n\x05SHELL\x10\x03\x12\x07\n\x03MCR\x10\x04')
+
+_LANGUAGE = descriptor.EnumDescriptor(
+ name='Language',
+ full_name='Language',
+ filename=None,
+ file=DESCRIPTOR,
+ values=[
+ descriptor.EnumValueDescriptor(
+ name='MATLAB', index=0, number=1,
+ options=None,
+ type=None),
+ descriptor.EnumValueDescriptor(
+ name='PYTHON', index=1, number=2,
+ options=None,
+ type=None),
+ descriptor.EnumValueDescriptor(
+ name='SHELL', index=2, number=3,
+ options=None,
+ type=None),
+ descriptor.EnumValueDescriptor(
+ name='MCR', index=3, number=4,
+ options=None,
+ type=None),
+ ],
+ containing_type=None,
+ options=None,
+ serialized_start=580,
+ serialized_end=634,
+)
+
+
+MATLAB = 1
+PYTHON = 2
+SHELL = 3
+MCR = 4
+
+
+_EXPERIMENT_PARAMETERSPEC_TYPE = descriptor.EnumDescriptor(
+ name='Type',
+ full_name='Experiment.ParameterSpec.Type',
+ filename=None,
+ file=DESCRIPTOR,
+ values=[
+ descriptor.EnumValueDescriptor(
+ name='INT', index=0, number=1,
+ options=None,
+ type=None),
+ descriptor.EnumValueDescriptor(
+ name='FLOAT', index=1, number=2,
+ options=None,
+ type=None),
+ descriptor.EnumValueDescriptor(
+ name='ENUM', index=2, number=3,
+ options=None,
+ type=None),
+ ],
+ containing_type=None,
+ options=None,
+ serialized_start=542,
+ serialized_end=578,
+)
+
+
+_JOB = descriptor.Descriptor(
+ name='Job',
+ full_name='Job',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ descriptor.FieldDescriptor(
+ name='id', full_name='Job.id', index=0,
+ number=1, type=4, cpp_type=4, label=2,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='expt_dir', full_name='Job.expt_dir', index=1,
+ number=2, type=9, cpp_type=9, label=2,
+ has_default_value=False, default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='name', full_name='Job.name', index=2,
+ number=3, type=9, cpp_type=9, label=2,
+ has_default_value=False, default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='language', full_name='Job.language', index=3,
+ number=4, type=14, cpp_type=8, label=2,
+ has_default_value=False, default_value=1,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='status', full_name='Job.status', index=4,
+ number=5, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='param', full_name='Job.param', index=5,
+ number=6, type=11, cpp_type=10, label=3,
+ has_default_value=False, default_value=[],
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='submit_t', full_name='Job.submit_t', index=6,
+ number=7, type=4, cpp_type=4, label=1,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='start_t', full_name='Job.start_t', index=7,
+ number=8, type=4, cpp_type=4, label=1,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='end_t', full_name='Job.end_t', index=8,
+ number=9, type=4, cpp_type=4, label=1,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='value', full_name='Job.value', index=9,
+ number=10, type=1, cpp_type=5, label=1,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='duration', full_name='Job.duration', index=10,
+ number=11, type=1, cpp_type=5, label=1,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ options=None,
+ is_extendable=False,
+ extension_ranges=[],
+ serialized_start=20,
+ serialized_end=224,
+)
+
+
+_PARAMETER = descriptor.Descriptor(
+ name='Parameter',
+ full_name='Parameter',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ descriptor.FieldDescriptor(
+ name='name', full_name='Parameter.name', index=0,
+ number=1, type=9, cpp_type=9, label=2,
+ has_default_value=False, default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='int_val', full_name='Parameter.int_val', index=1,
+ number=2, type=3, cpp_type=2, label=3,
+ has_default_value=False, default_value=[],
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='str_val', full_name='Parameter.str_val', index=2,
+ number=3, type=9, cpp_type=9, label=3,
+ has_default_value=False, default_value=[],
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='dbl_val', full_name='Parameter.dbl_val', index=3,
+ number=4, type=1, cpp_type=5, label=3,
+ has_default_value=False, default_value=[],
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ options=None,
+ is_extendable=False,
+ extension_ranges=[],
+ serialized_start=226,
+ serialized_end=302,
+)
+
+
+_EXPERIMENT_PARAMETERSPEC = descriptor.Descriptor(
+ name='ParameterSpec',
+ full_name='Experiment.ParameterSpec',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ descriptor.FieldDescriptor(
+ name='name', full_name='Experiment.ParameterSpec.name', index=0,
+ number=1, type=9, cpp_type=9, label=2,
+ has_default_value=False, default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='size', full_name='Experiment.ParameterSpec.size', index=1,
+ number=2, type=13, cpp_type=3, label=2,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='type', full_name='Experiment.ParameterSpec.type', index=2,
+ number=3, type=14, cpp_type=8, label=2,
+ has_default_value=False, default_value=1,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='options', full_name='Experiment.ParameterSpec.options', index=3,
+ number=4, type=9, cpp_type=9, label=3,
+ has_default_value=False, default_value=[],
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='min', full_name='Experiment.ParameterSpec.min', index=4,
+ number=5, type=1, cpp_type=5, label=1,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='max', full_name='Experiment.ParameterSpec.max', index=5,
+ number=6, type=1, cpp_type=5, label=1,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ _EXPERIMENT_PARAMETERSPEC_TYPE,
+ ],
+ options=None,
+ is_extendable=False,
+ extension_ranges=[],
+ serialized_start=408,
+ serialized_end=578,
+)
+
+_EXPERIMENT = descriptor.Descriptor(
+ name='Experiment',
+ full_name='Experiment',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ descriptor.FieldDescriptor(
+ name='language', full_name='Experiment.language', index=0,
+ number=1, type=14, cpp_type=8, label=2,
+ has_default_value=False, default_value=1,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='name', full_name='Experiment.name', index=1,
+ number=2, type=9, cpp_type=9, label=2,
+ has_default_value=False, default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='variable', full_name='Experiment.variable', index=2,
+ number=3, type=11, cpp_type=10, label=3,
+ has_default_value=False, default_value=[],
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ ],
+ extensions=[
+ ],
+ nested_types=[_EXPERIMENT_PARAMETERSPEC, ],
+ enum_types=[
+ ],
+ options=None,
+ is_extendable=False,
+ extension_ranges=[],
+ serialized_start=305,
+ serialized_end=578,
+)
+
+_JOB.fields_by_name['language'].enum_type = _LANGUAGE
+_JOB.fields_by_name['param'].message_type = _PARAMETER
+_EXPERIMENT_PARAMETERSPEC.fields_by_name['type'].enum_type = _EXPERIMENT_PARAMETERSPEC_TYPE
+_EXPERIMENT_PARAMETERSPEC.containing_type = _EXPERIMENT;
+_EXPERIMENT_PARAMETERSPEC_TYPE.containing_type = _EXPERIMENT_PARAMETERSPEC;
+_EXPERIMENT.fields_by_name['language'].enum_type = _LANGUAGE
+_EXPERIMENT.fields_by_name['variable'].message_type = _EXPERIMENT_PARAMETERSPEC
+DESCRIPTOR.message_types_by_name['Job'] = _JOB
+DESCRIPTOR.message_types_by_name['Parameter'] = _PARAMETER
+DESCRIPTOR.message_types_by_name['Experiment'] = _EXPERIMENT
+
+class Job(message.Message):
+ __metaclass__ = reflection.GeneratedProtocolMessageType
+ DESCRIPTOR = _JOB
+
+ # @@protoc_insertion_point(class_scope:Job)
+
+class Parameter(message.Message):
+ __metaclass__ = reflection.GeneratedProtocolMessageType
+ DESCRIPTOR = _PARAMETER
+
+ # @@protoc_insertion_point(class_scope:Parameter)
+
+class Experiment(message.Message):
+ __metaclass__ = reflection.GeneratedProtocolMessageType
+
+ class ParameterSpec(message.Message):
+ __metaclass__ = reflection.GeneratedProtocolMessageType
+ DESCRIPTOR = _EXPERIMENT_PARAMETERSPEC
+
+ # @@protoc_insertion_point(class_scope:Experiment.ParameterSpec)
+ DESCRIPTOR = _EXPERIMENT
+
+ # @@protoc_insertion_point(class_scope:Experiment)
+
+# @@protoc_insertion_point(module_scope)
diff --git a/format_converter/TpeSMAC.py b/HPOlib/format_converter/tpe_to_smac.py
similarity index 86%
rename from format_converter/TpeSMAC.py
rename to HPOlib/format_converter/tpe_to_smac.py
index 7da602c7..b1317448 100644
--- a/format_converter/TpeSMAC.py
+++ b/HPOlib/format_converter/tpe_to_smac.py
@@ -24,6 +24,8 @@
"""
+from hyperopt.pyll import base as base
+
""" Partly taken from https://github.com/jaberg/hyperopt/blob/smacout/hyperopt/smac.py """
@@ -33,8 +35,6 @@
from functools import partial
-import hyperopt
-
class Cond(object):
def __init__(self, name, val, op):
@@ -57,10 +57,10 @@ def __repr__(self):
EQ = partial(Cond, op='=')
-def expr_to_config(expr, conditions, hps):z
+def expr_to_config(expr, conditions, hps):
if conditions is None:
conditions = ()
- assert isinstance(expr, hyperopt.pyll.base.Apply)
+ assert isinstance(expr, base.Apply)
if expr.name == 'switch':
idx = expr.inputs()[0]
options = expr.inputs()[1:]
@@ -87,7 +87,7 @@ def expr_to_config(expr, conditions, hps):z
expr_to_config(ii, conditions, hps)
-def expr_to_smac_config(filename):
+def convert_tpe_to_smac_from_file(filename):
space_name, ext = os.path.splitext(os.path.basename(filename))
#noinspection PyBroadException
@@ -102,29 +102,30 @@ def expr_to_smac_config(filename):
search_space = space.space
- expr = hyperopt.pyll.as_apply(search_space)
+ expr = base.as_apply(search_space)
hps = {}
expr_to_config(expr, (True,), hps)
+ new_space = ""
# print expr
for label, dct in hps.items():
if dct['node'].name == "randint":
assert len(dct['node'].inputs()) == 1
#randint['x', 5] -> x [0, 4]i [0]
upper = dct['node'].inputs()[0].eval()
- print '%s {%s} [0]' % \
+ new_space += '%s {%s} [0]\n' % \
(label, ", ".join(["%s" % (i,) for i in range(upper)]))
elif dct['node'].name == "categorical":
# No difference to a randint node
upper = dct['node'].inputs()[1].eval()
- print '%s {%s} [0]' % \
- (label, ", ".join(["%s" % (i,) for i in range(upper)]))
+ new_space += '%s {%s} [0]\n' % \
+ (label, ", ".join(["%s" % (i,) for i in range(upper)]))
elif dct['node'].name == "uniform":
assert len(dct['node'].inputs()) == 2
lower = dct['node'].inputs()[0].eval()
upper = dct['node'].inputs()[1].eval()
default = (lower+upper)/2.0
- print '%s [%s, %s] [%s]' % (label, lower, upper, default)
+ new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, default)
elif dct['node'].name == "quniform":
# Assumption: q-value is always the last value
assert len(dct['node'].inputs()) == 3
@@ -133,14 +134,14 @@ def expr_to_smac_config(filename):
q = dct['node'].inputs()[2].eval()
default = (lower+upper)/2.0
label = "Q%s_%s" % (q, label)
- print '%s [%s, %s] [%s]' % (label, lower, upper, default)
+ new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, default)
elif dct['node'].name == "loguniform":
assert len(dct['node'].inputs()) == 2
lower = dct['node'].inputs()[0].eval()
upper = dct['node'].inputs()[1].eval()
default = (lower+upper)/2.0
label = "LOG_%s" % (label,)
- print '%s [%s, %s] [%s]' % (label, lower, upper, default)
+ new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, default)
elif dct['node'].name == "qloguniform":
assert len(dct['node'].inputs()) == 3
lower = dct['node'].inputs()[0].eval()
@@ -148,14 +149,14 @@ def expr_to_smac_config(filename):
q = dct['node'].inputs()[2].eval()
default = (lower+upper)/2.0
label = "LOG_Q%s_%s" % (q, label)
- print '%s [%s, %s] [%s]' % (label, lower, upper, default)
+ new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, default)
elif dct['node'].name == "normal":
assert len(dct['node'].inputs()) == 2
mu = dct['node'].inputs()[0].eval()
sigma = dct['node'].inputs()[1].eval()
lower = mu-3*sigma
upper = mu+3*sigma
- print '%s [%s, %s] [%s]' % (label, lower, upper, mu)
+ new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, mu)
elif dct['node'].name == "qnormal":
assert len(dct['node'].inputs()) == 3
mu = dct['node'].inputs()[0].eval()
@@ -164,7 +165,7 @@ def expr_to_smac_config(filename):
upper = mu+3*sigma
q = dct['node'].inputs()[2].eval()
label = "Q%s_%s" % (q, label)
- print '%s [%s, %s] [%s]' % (label, lower, upper, mu)
+ new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, mu)
elif dct['node'].name == "lognormal":
assert len(dct['node'].inputs()) == 2
mu = dct['node'].inputs()[0].eval()
@@ -172,7 +173,7 @@ def expr_to_smac_config(filename):
lower = mu-3*sigma
upper = mu+3*sigma
label = "LOG_%s" % (label, )
- print '%s [%s, %s] [%s]' % (label, lower, upper, mu)
+ new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, mu)
elif dct['node'].name == "qlognormal":
assert len(dct['node'].inputs()) == 3
mu = dct['node'].inputs()[0].eval()
@@ -181,7 +182,7 @@ def expr_to_smac_config(filename):
upper = mu+3*sigma
q = dct['node'].inputs()[2].eval()
label = "Q%s_LOG_%s" % (q, label)
- print '%s [%s, %s] [%s]' % (label, lower, upper, mu)
+ new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, mu)
else:
raise Exception("Node name %s not known" % dct['node'].name)
@@ -224,14 +225,8 @@ def expr_to_smac_config(filename):
# Start printing conditions
# print "CCC", varying_param, varying_param_name
if varying_param_name != "":
- print '%s | %s in {%s}' % (label, varying_param_name, ",".join(str(i) for i in varying_param))
+ new_space += '%s | %s in {%s}\n' % (label, varying_param_name, ",".join(str(i) for i in varying_param))
for key in condict.keys():
if key != varying_param_name:
- print '%s | %s in {%s}' % (label, key, condict[key])
-
-
-if __name__ == '__main__':
- if len(sys.argv) != 2:
- sys.stdout.write("Wrong number of Arguments\nUsage: python TpeSMAC.py \n")
- sys.exit(1)
- expr_to_smac_config(sys.argv[1])
+ new_space += '%s | %s in {%s}\n' % (label, key, condict[key])
+ return new_space
\ No newline at end of file
diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py
new file mode 100644
index 00000000..169ab626
--- /dev/null
+++ b/HPOlib/runsolver_wrapper.py
@@ -0,0 +1,372 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+from collections import OrderedDict
+import logging
+import numpy as np
+import os
+import re
+import subprocess
+import sys
+import time
+
+import HPOlib.Experiment as Experiment
+import HPOlib.wrapping_util as wrappingUtil
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+logging.basicConfig(format='[%(levelname)s] [%(asctime)s:%(name)s] %('
+ 'message)s', datefmt='%H:%M:%S')
+hpolib_logger = logging.getLogger("HPOlib")
+hpolib_logger.setLevel(logging.INFO)
+logger = logging.getLogger("HPOlib.runsolver_wrapper")
+
+
+# TODO: This should be in a util function sometime in the future
+# Is duplicated in cv.py
+def get_optimizer():
+ return "_".join(os.getcwd().split("/")[-1].split("_")[0:-2])
+
+
+def remove_param_metadata(params):
+ """
+ Check whether some params are defined on the Log scale or with a Q value,
+ must be marked with "LOG$_{paramname}" or Q[0-999]_$paramname
+ LOG/Q will be removed from the paramname
+ """
+ for para in params:
+ new_name = para
+ if "LOG10_" in para:
+ pos = para.find("LOG10")
+ new_name = para[0:pos] + para[pos + 6:]
+ # new_name = new_name.strip("_")
+ params[new_name] = np.power(10, float(params[para]))
+ del params[para]
+ elif "LOG2" in para:
+ pos = para.find("LOG2_")
+ new_name = para[0:pos] + para[pos + 5:]
+ # new_name = new_name.strip("_")
+ params[new_name] = np.power(2, float(params[para]))
+ del params[para]
+ elif "LOG_" in para:
+ pos = para.find("LOG")
+ new_name = para[0:pos] + para[pos + 4:]
+ # new_name = new_name.strip("_")
+ params[new_name] = np.exp(float(params[para]))
+ del params[para]
+ #Check for Q value, returns round(x/q)*q
+ m = re.search(r'Q[0-999\.]{1,10}_', para)
+ if m is not None:
+ pos = new_name.find(m.group(0))
+ tmp = new_name[0:pos] + new_name[pos + len(m.group(0)):]
+ #tmp = tmp.strip("_")
+ q = float(m.group(0)[1:-1])
+ params[tmp] = round(float(params[new_name]) / q) * q
+ del params[new_name]
+
+
+def load_experiment_file():
+ optimizer = get_optimizer()
+ experiment = Experiment.Experiment(".", optimizer)
+ return experiment
+
+
+def read_runsolver_output(runsolver_output_file):
+ """
+ Read the runsolver output, watch out for
+ Mem limit exceeded: sending SIGTERM then SIGKILL
+ Maximum CPU time exceeded: sending SIGTERM then SIGKILL
+ Maximum wall clock time exceeded: sending SIGTERM then SIGKILL
+ Maximum VSize exceeded: sending SIGTERM then SIGKILL
+ In case one of these happened, send back the worst possible result
+ as specified in the config
+ """
+ limit_exceeded = None
+ cpu_time = None
+ wallclock_time = None
+ solver_ended_section = False
+ with open(runsolver_output_file, 'r') as f:
+ runsolver_output_content = f.readlines()
+ for line in runsolver_output_content:
+ if "Maximum CPU time exceeded" in line:
+ limit_exceeded = "CPU time exceeded"
+ if "Maximum wall clock time exceeded" in line:
+ limit_exceeded = "Wall clock time exceeded"
+ if "Maximum VSize exceeded" in line:
+ limit_exceeded = "VSize exceeded"
+ if "Mem limit exceeded" in line:
+ limit_exceeded = "Memory exceeded"
+ if "Solver just ended. Dumping a history of the last" in line:
+ solver_ended_section = True
+ if re.search(r"Real time \(s\): ", line) and solver_ended_section:
+ wallclock_time = float(line.split()[3])
+ if re.search(r"^CPU time \(s\): ", line) and solver_ended_section:
+ cpu_time = float(line.split()[3])
+ return cpu_time, wallclock_time, limit_exceeded
+
+
+def read_run_instance_output(run_instance_output):
+ """
+ Read the run_instance output file
+ """
+ result_array = None
+ fh = open(run_instance_output, "r")
+ run_instance_content = fh.readlines()
+ fh.close()
+ result_string = None
+ for line in run_instance_content:
+ match = re.search(r"\s*[Rr]esult\s+(?:([Ff]or)|([oO]f))\s"
+ r"+(?:(HAL)|(ParamILS)|(SMAC)|([tT]his [wW]rapper))",
+ line)
+ if match:
+ pos = match.start(0)
+ result_string = line[pos:].strip()
+ result_array = result_string.split()
+ result_array = [value.strip(",") for value in result_array]
+ break
+
+ # If we do not find a result string, return the last three lines of the
+ # run_instance output
+ # TODO: there must be some better way to tell the user what happend
+ if not result_string and len(run_instance_content) >= 3:
+ result_string = "".join(run_instance_content[-3:])
+
+ return result_array, result_string
+
+
+def get_trial_index(experiment, fold, params):
+ # Check whether we are in a new configuration; This has to check whether
+ # the params were already inserted but also whether the fold already run
+ # This is checked twice; the instance_result has to be not NaN and the
+ # entry in instance_order has to exist
+ new = True
+ trial_index = np.NaN
+ for idx, trial in enumerate(experiment.trials):
+ exp = trial['params']
+ if exp == params and (idx, fold) not in experiment.instance_order and \
+ (experiment.get_trial_from_id(idx)['instance_results'][fold] == np.NaN or
+ experiment.get_trial_from_id(idx)['instance_results'][fold] !=
+ experiment.get_trial_from_id(idx)['instance_results'][fold]):
+ new = False
+ trial_index = idx
+ break
+ if new:
+ trial_index = experiment.add_job(params)
+ return trial_index
+
+
+def parse_command_line():
+ # Parse options and arguments
+ usage = "This script pickles the params and runs the runsolver with " + \
+ "run_instance and extract the output for the optimizer \n" + \
+ "The output is printed im a SMACish way: \n\n" + \
+ "'Result for ParamILS: , , , " + \
+ ", , ' \n\n" + \
+ "Usage: runsolver_wrapper " + \
+ " " + \
+ " \n" + \
+ " might be the optimizer name if not" + \
+ " called by smac\n"
+ if len(sys.argv) < 7:
+ sys.stdout.write(usage)
+ exit(1)
+
+ # Then get some information for run_instance
+ fold = int(sys.argv[1])
+ seed = int(sys.argv[5])
+ return fold, seed
+
+
+def make_command(cfg, fold, param_string, run_instance_output):
+ fn = cfg.get("HPOLIB", "function")
+ python_cmd = cfg.get("HPOLIB", "leading_algo_info") + " " + fn
+ python_cmd += " --fold %d --folds %d --params %s" % (fold, cfg.getint(
+ "HPOLIB", "number_cv_folds"), param_string)
+ # Do not write the actual task in quotes because runsolver will not work
+ # then; also we need use-pty and timestamp so that the "solver" output
+ # is flushed to the output directory
+ delay = 0
+ cmd = cfg.get("HPOLIB", "leading_runsolver_info")
+ cmd += " runsolver -o %s --timestamp --use-pty" % run_instance_output
+ if cfg.get('HPOLIB', 'runsolver_time_limit'):
+ cmd += " -W %d" % cfg.getint('HPOLIB', 'runsolver_time_limit')
+ if cfg.get('HPOLIB', 'cpu_limit'):
+ cmd += " -C %d" % cfg.getint('HPOLIB', 'cpu_limit')
+ if cfg.get('HPOLIB', 'memory_limit'):
+ cmd += " -M %d" % cfg.getint('HPOLIB', 'memory_limit')
+ if delay is not None:
+ cmd += " -d %d" % int(delay)
+ cmd += " " + python_cmd
+ return cmd
+
+
+def get_parameters():
+ params = dict(zip(sys.argv[6::2], sys.argv[7::2]))
+ remove_param_metadata(params)
+ params = OrderedDict(sorted(params.items(), key=lambda t: t[0]))
+ return params
+
+
+def parse_output_files(cfg, run_instance_output, runsolver_output_file):
+ cpu_time, measured_wallclock_time, error = read_runsolver_output(
+ runsolver_output_file)
+ result_array, result_string = read_run_instance_output(run_instance_output)
+ if not result_array:
+ logger.critical("We could not find anything matching our regexp. "
+ "Setting the target algorithm runtime to the time "
+ "measured by the runsolver. Last lines of your "
+ "output:\n%s"
+ % result_string)
+ instance_wallclock_time = measured_wallclock_time
+ result_array = [None]*7
+ else:
+ instance_wallclock_time = float(result_array[4])
+
+ if cfg.getboolean("HPOLIB", "use_own_time_measurement") is True:
+ wallclock_time = measured_wallclock_time
+ else:
+ wallclock_time = instance_wallclock_time
+
+ if error is None and result_string is None:
+ additional_data = "No result string returned. Please have a look " \
+ "at " + run_instance_output
+ rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("HPOLIB",
+ "result_on_terminate"), additional_data)
+ os.remove(runsolver_output_file)
+
+ elif error is None and result_array[3] != "SAT":
+ additional_data = "Please have a look at " + run_instance_output + "." \
+ "The output status is not \"SAT\""
+ rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("HPOLIB",
+ "result_on_terminate"), additional_data)
+ os.remove(runsolver_output_file)
+
+ elif error is None and not np.isfinite(float(result_array[6].strip(","))):
+ additional_data = "Response value is not finite. Please have a look " \
+ "at " + run_instance_output
+ rval = (cpu_time, wallclock_time, "UNSAT", cfg.getfloat("HPOLIB",
+ "result_on_terminate"), additional_data)
+
+ elif error is None:
+ if cfg.getboolean("HPOLIB", "remove_target_algorithm_output"):
+ os.remove(run_instance_output)
+ os.remove(runsolver_output_file)
+ rval = (cpu_time, wallclock_time, "SAT", float(result_array[6].strip(",")),
+ cfg.get("HPOLIB", "function"))
+
+ else:
+ rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("HPOLIB",
+ "result_on_terminate"),
+ error + " Please have a look at " +
+ runsolver_output_file)
+ # It is useful to have the run_instance_output for debugging
+ os.remove(run_instance_output)
+
+ return rval
+
+
+def format_return_string(status, runtime, runlength, quality, seed,
+ additional_data):
+ return_string = "Result for ParamILS: %s, %f, %d, %f, %d, %s" % \
+ (status, runtime, runlength, quality, seed, additional_data)
+ return return_string
+
+
+def main():
+ """
+ If we are not called from cv means we are called from CLI. This means
+ the optimizer itself handles crossvalidation (smac). To keep a nice .pkl we have to do some
+ bookkeeping here
+ """
+
+ cfg = wrappingUtil.load_experiment_config_file()
+ called_from_cv = True
+ if cfg.getint('HPOLIB', 'handles_cv') == 1:
+ # If Our Optimizer can handle crossvalidation,
+ # we are called from CLI. To keep a sane nice .pkl
+ # we have to do some bookkeeping here
+ called_from_cv = False
+
+ # This has to be done here for SMAC, since smac does not call cv.py
+ if not called_from_cv:
+ cv_starttime = time.time()
+ experiment = load_experiment_file()
+ experiment.start_cv(cv_starttime)
+ del experiment
+
+ fold, seed = parse_command_line()
+ # Side-effect: removes all additional information like log and applies
+ # transformations to the parameters
+ params = get_parameters()
+ param_string = " ".join([key + " " + str(params[key]) for key in params])
+
+ time_string = wrappingUtil.get_time_string()
+ run_instance_output = os.path.join(os.getcwd(),
+ time_string + "_run_instance.out")
+ runsolver_output_file = os.path.join(os.getcwd(),
+ time_string + "_runsolver.out")
+
+ cmd = make_command(cfg, fold, param_string, run_instance_output)
+
+ fh = open(runsolver_output_file, "w")
+ experiment = load_experiment_file()
+ # Side-effect: adds a job if it is not yet in the experiments file
+ trial_index = get_trial_index(experiment, fold, params)
+ experiment.set_one_fold_running(trial_index, fold)
+ del experiment # release Experiment lock
+ logger.debug("Calling: %s" % cmd)
+ process = subprocess.Popen(cmd, stdout=fh,
+ stderr=fh, shell=True, executable="/bin/bash")
+
+ logger.info("-----------------------RUNNING RUNSOLVER----------------------------")
+ process.wait()
+ fh.close()
+
+ cpu_time, wallclock_time, status, result, additional_data = \
+ parse_output_files(cfg, run_instance_output, runsolver_output_file)
+
+ experiment = load_experiment_file()
+ if status == "SAT":
+ experiment.set_one_fold_complete(trial_index, fold, result,
+ wallclock_time)
+ elif status == "CRASHED" or status == "UNSAT":
+ result = cfg.getfloat("HPOLIB", "result_on_terminate")
+ experiment.set_one_fold_crashed(trial_index, fold, result,
+ wallclock_time)
+ status = "SAT"
+ else:
+ # TODO: We need a global stopping mechanism
+ pass
+ del experiment # release lock
+
+ return_string = format_return_string(status, wallclock_time, 1, result,
+ seed, additional_data)
+
+ if not called_from_cv:
+ experiment = load_experiment_file()
+ experiment.end_cv(time.time())
+ del experiment
+
+ logger.info(return_string)
+ print return_string
+ return return_string
+
+
+if __name__ == "__main__":
+ main()
diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py
new file mode 100644
index 00000000..c4bbc886
--- /dev/null
+++ b/HPOlib/wrapping.py
@@ -0,0 +1,406 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+from argparse import ArgumentParser
+import imp
+import logging
+import os
+from Queue import Queue, Empty
+import signal
+import shlex
+import subprocess
+import sys
+from threading import Thread
+import time
+
+import HPOlib
+import HPOlib.check_before_start as check_before_start
+import HPOlib.wrapping_util as wrapping_util
+# Experiment is imported after we check for numpy
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+logging.basicConfig(format='[%(levelname)s] [%(asctime)s:%(name)s] %('
+ 'message)s', datefmt='%H:%M:%S')
+hpolib_logger = logging.getLogger("HPOlib")
+hpolib_logger.setLevel(logging.INFO)
+logger = logging.getLogger("HPOlib.wrapping")
+
+
+def calculate_wrapping_overhead(trials):
+ wrapping_time = 0
+ for times in zip(trials.cv_starttime, trials.cv_endtime):
+ wrapping_time += times[1] - times[0]
+
+ # We need to import numpy again
+ import numpy as np
+ benchmark_time = 0
+ for t in trials.trials:
+ benchmark_time += np.nansum(t['instance_durations'])
+ wrapping_time = wrapping_time - benchmark_time
+ return wrapping_time
+
+
+def calculate_optimizer_time(trials):
+ optimizer_time = []
+ time_idx = 0
+
+ optimizer_time.append(trials.cv_starttime[0] - trials.starttime[time_idx])
+
+ for i in range(len(trials.cv_starttime[1:])):
+ if trials.cv_starttime[i + 1] > trials.endtime[time_idx]:
+ optimizer_time.append(trials.endtime[time_idx] -
+ trials.cv_endtime[i])
+ time_idx += 1
+ optimizer_time.append(trials.cv_starttime[i + 1] -
+ trials.starttime[time_idx])
+ else:
+ optimizer_time.append(trials.cv_starttime[i + 1] -
+ trials.cv_endtime[i])
+
+ optimizer_time.append(trials.endtime[time_idx] - trials.cv_endtime[-1])
+ trials.optimizer_time = optimizer_time
+
+ # We need to import numpy again
+ import numpy as np
+ return np.nansum(optimizer_time)
+
+
+def use_arg_parser():
+ """Parse all options which can be handled by the wrapping script.
+ Unknown arguments are ignored and returned as a list. It is useful to
+ check this list in your program to handle typos etc.
+
+ Returns:
+ a tuple. The first element is an argparse.Namespace object,
+ the second a list with all unknown arguments.
+ """
+ description = "Perform an experiment with HPOlib. " \
+ "Call this script from experiment directory (containing 'config.cfg')"
+ epilog = "Your are using HPOlib " + HPOlib.__version__
+ prog = "path/from/Experiment/to/HPOlib/wrapping.py"
+
+ parser = ArgumentParser(description=description, prog=prog, epilog=epilog)
+
+ parser.add_argument("-o", "--optimizer", action="store", type=str,
+ dest="optimizer",
+ help="Specify the optimizer name.", required=True)
+ parser.add_argument("-p", "--print", action="store_true",
+ dest="printcmd", default=False,
+ help="If set print the command instead of executing it")
+ parser.add_argument("-s", "--seed", action="store", type=int,
+ dest="seed", default=1,
+ help="Set the seed of the optimizer")
+ parser.add_argument("-t", "--title", action="store", type=str,
+ dest="title", default=None,
+ help="A title for the experiment")
+ parser.add_argument("--cwd", action="store", type=str, dest="working_dir",
+ default=None, help="Change the working directory to "
+ " prior to running the experiment")
+ parser.add_argument("-r", "--restore", action="store", type=str,
+ dest="restore", default=None,
+ help="Restore the state from a given directory")
+
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument("-q", "--silent", action="store_true",
+ dest="silent", default=False,
+ help="Don't print anything during optimization")
+ group.add_argument("-v", "--verbose", action="store_true",
+ dest="verbose", default=False,
+ help="Print stderr/stdout for optimizer")
+
+ args, unknown = parser.parse_known_args()
+ return args, unknown
+
+
+def main():
+ """Start an optimization of the HPOlib. For documentation see the
+ comments inside this function and the general HPOlib documentation."""
+ args, unknown_arguments = use_arg_parser()
+
+ # Convert the path to the optimizer to be an absolute path, which is
+ # necessary later when we change the working directory
+ optimizer = args.optimizer
+ if not os.path.isabs(optimizer):
+ relative_path = optimizer
+ optimizer = os.path.abspath(optimizer)
+ logger.info("Converting relative optimizer path %s to absolute "
+ "optimizer path %s.", relative_path, optimizer)
+
+ if args.working_dir:
+ os.chdir(args.working_dir)
+
+ experiment_dir = os.getcwd()
+ check_before_start.check_first(experiment_dir)
+
+ # Now we can safely import non standard things
+ import numpy as np
+ import HPOlib.Experiment as Experiment # Wants numpy and scipy
+
+ # Check how many optimizer versions are present and if all dependencies
+ # are installed
+ optimizer_version = check_before_start.check_optimizer(optimizer)
+
+ logger.warning("You called -o %s, I am using optimizer defined in "
+ "%sDefault.cfg", optimizer, optimizer_version)
+ optimizer = os.path.basename(optimizer_version)
+
+ config = wrapping_util.get_configuration(experiment_dir,
+ optimizer_version, unknown_arguments)
+
+ # Saving the config file is down further at the bottom, as soon as we get
+ # hold of the new optimizer directory
+ wrapping_dir = os.path.dirname(os.path.realpath(__file__))
+
+ # Load optimizer
+ try:
+ optimizer_dir = os.path.dirname(os.path.realpath(optimizer_version))
+ optimizer_module = imp.load_source(optimizer_dir, optimizer_version + ".py")
+ except (ImportError, IOError):
+ logger.critical("Optimizer module %s not found", optimizer)
+ import traceback
+ logger.critical(traceback.format_exc())
+ sys.exit(1)
+ optimizer_call, optimizer_dir_in_experiment = optimizer_module.main(config=config,
+ options=args,
+ experiment_dir=experiment_dir)
+ cmd = optimizer_call
+
+ with open(os.path.join(optimizer_dir_in_experiment, "config.cfg"), "w") as f:
+ config.set("HPOLIB", "is_not_original_config_file", "True")
+ wrapping_util.save_config_to_file(f, config, write_nones=True)
+
+ # initialize/reload pickle file
+ if args.restore:
+ try:
+ os.remove(os.path.join(optimizer_dir_in_experiment, optimizer + ".pkl.lock"))
+ except OSError:
+ pass
+ folds = config.getint('HPOLIB', 'number_cv_folds')
+ trials = Experiment.Experiment(optimizer_dir_in_experiment, optimizer, folds=folds,
+ max_wallclock_time=config.get('HPOLIB',
+ 'cpu_limit'),
+ title=args.title)
+ trials.optimizer = optimizer_version
+
+ if args.restore:
+ #noinspection PyBroadException
+ try:
+ restored_runs = optimizer_module.restore(config=config,
+ optimizer_dir=optimizer_dir_in_experiment,
+ cmd=cmd)
+ except:
+ logger.critical("Could not restore runs for %s", args.restore)
+ import traceback
+ logger.critical(traceback.format_exc())
+ sys.exit(1)
+
+ logger.info("Restored %d runs", restored_runs)
+ trials.remove_all_but_first_runs(restored_runs)
+ fh = open(os.path.join(optimizer_dir_in_experiment, optimizer + ".out"), "a")
+ fh.write("#" * 80 + "\n" + "Restart! Restored %d runs.\n" % restored_runs)
+ fh.close()
+
+ if len(trials.endtime) < len(trials.starttime):
+ trials.endtime.append(trials.cv_endtime[-1])
+ trials.starttime.append(time.time())
+ else:
+ trials.starttime.append(time.time())
+ #noinspection PyProtectedMember
+ trials._save_jobs()
+ del trials
+ sys.stdout.flush()
+
+ # Run call
+ if args.printcmd:
+ logger.info(cmd)
+ return 0
+ else:
+ # call target_function.setup()
+ fn_setup = config.get("HPOLIB", "function_setup")
+ if fn_setup:
+ try:
+ output = subprocess.check_output(fn_setup, stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as e:
+ logger.critical(e.output)
+ sys.exit(1)
+
+ logger.info(cmd)
+ output_file = os.path.join(optimizer_dir_in_experiment, optimizer + ".out")
+ fh = open(output_file, "a")
+ cmd = shlex.split(cmd)
+ print cmd
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, preexec_fn=os.setsid)
+ logger.info("-----------------------RUNNING----------------------------------")
+ # http://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python
+ # How often is the experiment pickle supposed to be opened?
+ if config.get("HPOLIB", "total_time_limit"):
+ optimizer_end_time = time.time() + config.getint("HPOLIB", "total_time_limit")
+ else:
+ optimizer_end_time = sys.float_info.max
+
+ last_output = time.time()
+ printed_start_configuration = list()
+ printed_end_configuration = list()
+ current_best = -1
+ sent_SIGTERM = False
+ sent_SIGKILL = False
+ # After the evaluation finished, we scan the experiment pickle twice
+ # to print everything!
+ minimal_runs_to_go = 2
+
+ def enqueue_output(out, queue):
+ for line in iter(out.readline, b''):
+ queue.put(line)
+ out.close()
+
+ stderr_queue = Queue()
+ stdout_queue = Queue()
+ stderr_thread = Thread(target=enqueue_output, args=(proc.stderr, stderr_queue))
+ stdout_thread = Thread(target=enqueue_output, args=(proc.stdout, stdout_queue))
+ stderr_thread.daemon = True
+ stdout_thread.daemon = True
+ stderr_thread.start()
+ stdout_thread.start()
+ logger.info('Optimizer runs with PID: %d', proc.pid)
+
+ while minimal_runs_to_go > 0: # Think of this as a do-while loop...
+ try:
+ while True:
+ line = stdout_queue.get_nowait()
+ fh.write(line)
+
+ # Write to stdout only if verbose is on
+ if args.verbose:
+ sys.stdout.write(line)
+ sys.stdout.flush()
+ except Empty:
+ pass
+
+ try:
+ while True:
+ line = stderr_queue.get_nowait()
+ fh.write(line)
+
+ # Write always, except silent is on
+ if not args.silent:
+ sys.stderr.write("[ERR]:" + line)
+ sys.stderr.flush()
+ except Empty:
+ pass
+
+ if time.time() > optimizer_end_time and not sent_SIGTERM:
+ os.killpg(proc.pid, signal.SIGTERM)
+ sent_SIGTERM = True
+
+ if time.time() > optimizer_end_time + 200 and not sent_SIGKILL:
+ os.killpg(proc.pid, signal.SIGKILL)
+ sent_SIGKILL = True
+
+ fh.flush()
+ # necessary, otherwise HPOlib-run takes 100% of one processor
+ time.sleep(0.1)
+
+ if not (args.verbose or args.silent) and time.time() - last_output > 1:
+ trials = Experiment.Experiment(optimizer_dir_in_experiment,
+ optimizer)
+
+ for i in range(len(printed_end_configuration), len(trials.instance_order)):
+ configuration = trials.instance_order[i][0]
+ fold = trials.instance_order[i][1]
+ if i + 1 > len(printed_start_configuration):
+ logger.info("Starting configuration %5d, fold %2d",
+ configuration, fold)
+ printed_start_configuration.append(i)
+
+ if np.isfinite(trials.trials[configuration]
+ ["instance_results"][fold]):
+ last_result = trials.trials[configuration] \
+ ["instance_results"][fold]
+ tmp_current_best = trials.get_arg_best()
+ if tmp_current_best <= i:
+ current_best = tmp_current_best
+ # Calculate current best
+ # Check if last result is finite, if not calc nanmean over all instances
+ dct_helper = trials.trials[current_best]
+ res = dct_helper["result"] if \
+ np.isfinite(dct_helper["result"]) \
+ else wrapping_util.nan_mean(dct_helper["instance_results"])
+ #np.nanmean(trials.trials[current_best]["instance_results"])
+ # nanmean does not work for all numpy version
+ logger.info("Result %10f, current best %10f",
+ last_result, res)
+ printed_end_configuration.append(i)
+
+ del trials
+ last_output = time.time()
+
+ if proc.poll() is not None: # the end condition for the
+ minimal_runs_to_go -= 1 # do-while loop
+
+ elif args.verbose or args.silent:
+ if proc.poll() is not None:
+ minimal_runs_to_go -= 1
+
+ ret = proc.returncode
+
+ logger.info("-----------------------END--------------------------------------")
+ fh.close()
+
+ # call target_function.teardown()
+ fn_teardown = config.get("HPOLIB", "function_teardown")
+ if fn_teardown:
+ try:
+ output = subprocess.check_output(fn_teardown, stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as e:
+ logger.critical(e.output)
+ sys.exit(1)
+
+ trials = Experiment.Experiment(optimizer_dir_in_experiment, optimizer)
+ trials.endtime.append(time.time())
+ #noinspection PyProtectedMember
+ trials._save_jobs()
+ # trials.finish_experiment()
+ total_time = 0
+ logger.info("Best result")
+ logger.info(trials.get_best())
+ logger.info("Durations")
+ try:
+ for starttime, endtime in zip(trials.starttime, trials.endtime):
+ total_time += endtime - starttime
+ logger.info("Needed a total of %f seconds", total_time)
+ logger.info("The optimizer %s took %10.5f seconds",
+ optimizer, float(calculate_optimizer_time(trials)))
+ logger.info("The overhead of HPOlib is %f seconds",
+ calculate_wrapping_overhead(trials))
+ logger.info("The benchmark itself took %f seconds" % \
+ trials.total_wallclock_time)
+ except Exception as e:
+ logger.error(HPOlib.wrapping_util.format_traceback(sys.exc_info()))
+ logger.error("Experiment itself went fine, but calculating "
+ "durations of optimization failed: %s %s",
+ sys.exc_info()[0], e)
+ del trials
+ logger.info("Finished with return code: " + str(ret))
+ return ret
+
+if __name__ == "__main__":
+ main()
diff --git a/HPOlib/wrapping_util.py b/HPOlib/wrapping_util.py
new file mode 100644
index 00000000..30536998
--- /dev/null
+++ b/HPOlib/wrapping_util.py
@@ -0,0 +1,245 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+from argparse import ArgumentParser
+from ConfigParser import SafeConfigParser
+import datetime
+import logging
+import imp
+import math
+import numpy as np
+import traceback
+import os
+from StringIO import StringIO
+import sys
+
+import config_parser.parse as parse
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+logger = logging.getLogger("HPOlib.wrapping_util")
+
+
+def nan_mean(arr):
+ # First: Sum all finite elements
+ arr = np.array(arr)
+ res = sum([ele for ele in arr if np.isfinite(ele)])
+ num_ele = (arr.size - np.count_nonzero(~np.isfinite(arr)))
+ if num_ele == 0:
+ return np.nan
+ if num_ele != 0 and res == 0:
+ return 0
+ # Second: divide with number of finite elements
+ res /= num_ele
+ return res
+
+
+def get_time_string():
+ local_time = datetime.datetime.today()
+ time_string = "%d-%d-%d--%d-%d-%d-%d" % (local_time.year, local_time.month,
+ local_time.day, local_time.hour, local_time.minute,
+ local_time.second, local_time.microsecond)
+ return time_string
+
+
+def float_eq(a, b, eps=0.0001):
+ return abs(math.log(a+1) - math.log(b+1)) <= eps
+
+
+def format_traceback(exc_info):
+ traceback_template = '''Traceback (most recent call last):
+ File "%(filename)s", line %(lineno)s, in %(name)s
+ %(type)s: %(message)s\n''' # Skipping the "actual line" item
+
+ # Also note: we don't walk all the way through the frame stack in this example
+ # see hg.python.org/cpython/file/8dffb76faacc/Lib/traceback.py#l280
+ # (Imagine if the 1/0, below, were replaced by a call to test() which did 1/0.)
+
+ exc_type, exc_value, exc_traceback = exc_info # most recent (if any) by default
+
+ '''
+ Reason this _can_ be bad: If an (unhandled) exception happens AFTER this,
+ or if we do not delete the labels on (not much) older versions of Py, the
+ reference we created can linger.
+
+ traceback.format_exc/print_exc do this very thing, BUT note this creates a
+ temp scope within the function.
+ '''
+
+ traceback_details = {
+ 'filename': exc_traceback.tb_frame.f_code.co_filename,
+ 'lineno' : exc_traceback.tb_lineno,
+ 'name' : exc_traceback.tb_frame.f_code.co_name,
+ 'type' : exc_type.__name__,
+ 'message' : exc_value.message, # or see traceback._some_str()
+ }
+
+ del(exc_type, exc_value, exc_traceback) # So we don't leave our local labels/objects dangling
+ # This still isn't "completely safe", though!
+ # "Best (recommended) practice: replace all exc_type, exc_value, exc_traceback
+ # with sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]
+
+ return "\n" + traceback.format_exc() + "\n\n" + traceback_template % traceback_details + "\n\n"
+
+
+def load_experiment_config_file():
+ # Load the config file, this holds information about data, black box fn etc.
+ try:
+ cfg_filename = "config.cfg"
+ config = SafeConfigParser(allow_no_value=True)
+ config.read(cfg_filename)
+ if not config.has_option("HPOLIB", "is_not_original_config_file"):
+ logger.critical("Config file in directory %s seems to be an"
+ " original config which was not created by wrapping.py. "
+ "Please contact the HPOlib maintainer to solve this issue.")
+ sys.exit(1)
+ return config
+ except IOError as e:
+ logger.critical("Could not open config file in directory %s",
+ os.getcwd())
+ sys.exit(1)
+
+
+def get_configuration(experiment_dir, optimizer_version, unknown_arguments):
+ """How the configuration is parsed:
+ 1. The command line is already parsed, we have a list of unknown arguments
+ 2. Assemble list of all config files related to this experiment, these are
+ 1. the general default config
+ 2. the optimizer default config
+ 3. the experiment configuration
+ 3. call parse_config to find out all allowed keys. Ignore the values.
+ 4. call parse_config_values_from_unknown_arguments to get parameters for
+ the optimization software from the command line in a config object
+ 5. call parse_config again. Read config files in the order specified
+ under point 2. Then read the command line arguments.
+ """
+ config_files = list()
+ general_default = os.path.join(os.path.dirname(parse.__file__),
+ "generalDefault.cfg")
+ config_files.append(general_default)
+ # Load optimizer parsing module
+ if optimizer_version != "" and optimizer_version is not None:
+ optimizer_version_parser = optimizer_version + "_parser"
+ # If optimizer_version_parser is an absolute path, the path of
+ # __file__ will be ignored
+ optimizer_version_parser_path = os.path.join(
+ os.path.dirname(os.path.realpath(__file__)),
+ optimizer_version_parser + ".py")
+ # noinspection PyBroadException,PyUnusedLocal
+ try:
+ optimizer_module_parser = imp.load_source(optimizer_version_parser,
+ optimizer_version_parser_path)
+ except Exception as e:
+ logger.critical('Could not find\n%s\n\tin\n%s\n\t relative to\n%s',
+ optimizer_version_parser,
+ optimizer_version_parser_path, os.getcwd())
+ import traceback
+
+ logger.critical(traceback.format_exc())
+ sys.exit(1)
+
+ optimizer_config_fn = os.path.splitext(optimizer_module_parser
+ .__file__)[0][:-7] + "Default.cfg"
+ if not os.path.exists(optimizer_config_fn):
+ logger.critical("No default config %s found", optimizer_config_fn)
+ sys.exit(1)
+ config_files.append(optimizer_config_fn)
+
+ path_to_current_config = os.path.join(experiment_dir, "config.cfg")
+ if os.path.exists(path_to_current_config):
+ config_files.append(os.path.join(experiment_dir, "config.cfg"))
+ else:
+ logger.info("No config file found. Only considering CLI arguments.")
+ #TODO Is the config file really the right place to get the allowed keys for a hyperparameter optimizer?
+ config = parse.parse_config(config_files, allow_no_value=True,
+ optimizer_version=optimizer_version)
+
+ if unknown_arguments is not None:
+ # Parse command line arguments
+ config_overrides = parse_config_values_from_unknown_arguments(
+ unknown_arguments, config)
+ # Remove every option from the configuration which was not present on the
+ # command line
+ config = config_with_cli_arguments(config, config_overrides)
+ # Transform to a string so we can pass it to parse_config
+ fh = StringIO()
+ save_config_to_file(fh, config, write_nones=False)
+ fh.seek(0)
+ else:
+ fh = None
+ config = parse.parse_config(config_files, allow_no_value=True,
+ optimizer_version=optimizer_version,
+ cli_values=fh)
+ if fh is not None:
+ fh.close()
+ if optimizer_version != "" and optimizer_version is not None:
+ config = optimizer_module_parser.manipulate_config(config)
+
+ # Check whether we have all necessary options
+ parse.check_config(config)
+ return config
+
+
+def parse_config_values_from_unknown_arguments(unknown_arguments, config):
+ """Parse values not recognized by use_option_parser for config values.
+
+ Args:
+ unknown_arguments: A list of arguments which is returned by
+ use_option_parser. It should only contain keys which are allowed
+ in config files.
+ config: A ConfigParser.SafeConfigParser object which contains all keys
+ should be parsed from the unknown_arguments list.
+ Returns:
+ an argparse.Namespace object containing the parsed values.
+ Raises:
+ an error if an argument from unknown_arguments is not a key in config
+ """
+ further_possible_command_line_arguments = []
+ for section in config.sections():
+ for key in config.options(section):
+ further_possible_command_line_arguments.append("--" + section +
+ ":" + key)
+
+ parser = ArgumentParser()
+ for argument in further_possible_command_line_arguments:
+ parser.add_argument(argument)
+
+ return parser.parse_args(unknown_arguments)
+
+
+def config_with_cli_arguments(config, config_overrides):
+ arg_dict = vars(config_overrides)
+ for section in config.sections():
+ for key in config.options(section):
+ cli_key = "%s:%s" % (section, key)
+ if cli_key in arg_dict:
+ config.set(section, key, arg_dict[cli_key])
+ else:
+ config.remove_option(section, key)
+ return config
+
+
+def save_config_to_file(file_handle, config, write_nones=True):
+ for section in config.sections():
+ file_handle.write("[" + section + "]\n")
+ for key in config.options(section):
+ if (config.get(section, key) is None and write_nones) or \
+ config.get(section, key) is not None:
+ file_handle.write("%s = %s\n" % (key, config.get(section, key)))
\ No newline at end of file
diff --git a/INSTALL.md b/INSTALL.md
new file mode 100644
index 00000000..212f6492
--- /dev/null
+++ b/INSTALL.md
@@ -0,0 +1,182 @@
+INSTALLATION INSTRUCTIONS FOR HPOlib
+-------
+
+``git clone https://github.com/automl/HPOlib.git``
+
+Installing inside an virtualenv
+-----------
+
+1.) Get [virtualenv](http://www.virtualenv.org/en/latest/virtualenv.html#installation), then load a freshly created virtualenv. (If you are not familiar with virtualenv, you might want to read [more](http://www.virtualenv.org/en/latest/virtualenv.html) about it)
+
+```bash
+pip install virtualenv
+virtualenv virtualHPOlib
+source virtualHPOlib/bin/activate
+```
+
+3.) Install ``numpy``, ``scipy``, ``matplotlib``, as this doesn't work through setup.py
+```bash
+easy_install -U distribute
+pip install numpy
+pip install scipy==0.13.2
+pip install matplotlib
+```
+This may take some time. Afterwards you can verify having those libs installed with:
+
+```bash
+pip freeze
+ argparse==1.2.1
+ backports.ssl-match-hostname==3.4.0.2
+ distribute==0.7.3
+ matplotlib==1.3.1
+ nose==1.3.0
+ numpy==1.8.0
+ pyparsing==2.0.1
+ python-dateutil==2.2
+ scipy==0.13.2
+ six==1.5.2
+ tornado==3.2
+ wsgiref==0.1.2
+```
+
+4.) run setup.py
+
+```python
+ python setup.py install
+```
+
+This will install HPOlib and some requirements (``networkx``, ``protobuf``, ``pymongo``).
+Be sure your system is **connected to the internet**, so ``setup.py`` can download
+optimizer and runsolver code. Your environment now looks like that
+
+```bash
+pip freeze
+ HPOlib==0.0.1
+ argparse==1.2.1
+ backports.ssl-match-hostname==3.4.0.2
+ distribute==0.7.3
+ matplotlib==1.3.1
+ networkx==1.8.1
+ nose==1.3.0
+ numpy==1.8.0
+ protobuf==2.5.0
+ pymongo==2.6.3
+ pyparsing==2.0.1
+ python-dateutil==2.2
+ scipy==0.13.3
+ six==1.5.2
+ tornado==3.2
+ wsgiref==0.1.2
+```
+
+and
+
+```bash
+ls optimizers/smac
+ smac_2_06_01-dev_parser.py smac_2_06_01-dev.py smac_2_06_01-dev_src smac_2_06_01-devDefault.cfg
+```
+
+5.) You can now run, e.g. smac with 200 evaluations on the branin function:
+
+```bash
+cd benchmarks/branin
+HPOlib-run -o ../../optimizers/smac/sma -s 23
+```
+
+This takes depending on your machine ~2 minutes. You can now plot the results of your first experiment:
+
+```bash
+HPOlib-plot FIRSTRUN smac_2_06_01-dev_23_*/smac_*.pkl -s `pwd`/Plots/
+```
+
+You can test the other optimizers (spearmint will take quite longer 30min):
+
+```bash
+ HPOlib-run -o ../../optimizers/tpe/h -s 23
+ HPOlib-run -o ../../optimizers/spearmint/s -s 23
+```
+
+and again:
+
+```bash
+ HPOlib-plot SMAC smac_2_06_01-dev_23_*/smac_*.pkl TPE hyperopt_august2013_mod_23_*/hyp*.pkl SPEARMINT spearmint_april2013_mod_23_*/spear*.pkl -s `pwd`/Plots/
+and to check the general performance on this super complex benchmark:
+
+ HPOlib-plot BRANIN smac_2_06_01-dev_23_*/smac_*.pkl hyperopt_august2013_mod_23_*/hyp*.pkl spearmint_april2013_mod_23_*/spear*.pkl -s `pwd`/Plots/
+```
+
+Using without installation
+-----------
+
+
+If you decide to not install HPOlib, you need to download the optimizer code by yourself
+
+```bash
+ cd optimizers
+ wget http://www.automl.org/hyperopt_august2013_mod_src.tar.gz
+ wget http://www.automl.org/smac_2_06_01-dev_src.tar.gz
+ wget http://www.automl.org/spearmint_april2013_mod_src.tar.gz
+
+ tar -xf hyperopt_august2013_mod_src.tar.gz
+ mv hyperopt_august2013_mod_src tpe/
+
+ tar -xf smac_2_06_01-dev_src.tar.gz
+ mv smac_2_06_01-dev_src.tar.gz smac/
+
+ tar -xf spearmint_april2013_mod_src.tar.gz
+ mv spearmint_april2013_mod_src spearmint/
+
+ cd ../
+```
+
+And you need to install all requirements:
+
+* numpy
+* matplotlib
+* networkx
+* protobuf
+* scipy
+* pymongo
+
+e.g. with
+
+```bash
+ sudo apt-get install python-numpy python-scipy mongodb python-networkx python-protobuf
+```
+
+Also you need the runsolver
+
+```bash
+ wget http://www.cril.univ-artois.fr/~roussel/runsolver/runsolver-3.3.2.tar.bz2
+ tar -xf runsolver-3.3.2.tar.bz2
+ cd runsolver/src
+ make
+```
+
+as this might not work, you can change the makefile via
+
+```bash
+ sed -i 's/\/usr\/include\/asm\/unistd/\/usr\/include\/unistd/g' ./Makefile
+ make
+```
+
+then you need to add runsolver (and HPOlib) to your PATH (PYTHONPATH):
+
+```bash
+ cd ../../
+ export PATH=$PATH:/path/to/runsolver/src/
+ export PYTHONPATH=$PYTHONPATH:`pwd`
+```
+
+then you can run a benchmark like in step 5.) from installing with setup.py with replacing
+``HPOlib-run`` with ``../../scripts/HPOlib-run`` and ``HPOlib-plot`` with ``../../scripts/HPOlib-plot``
+
+**FOR FURTHER DETAILS VISIT:** www.automl.org
+
+
+**Problems during installation**
+
+``python setup.py`` crashes with ``ImportError: cannot import name Feature`` during installing pymongo. This happens due to pymongo using a deprecated feature ''Feature'', which is not available in the setuptools version (>2.2). This error is fixed, but not yet available on PYPI.
+
+Solution: Downgrade ``setuptools`` with ``pip install setuptools==2.2`` and try again or install ``pymongo`` manually.
+
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 00000000..91f371be
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1 @@
+include *.md
\ No newline at end of file
diff --git a/Plotting/doAllPlots.sh b/Plotting/doAllPlots.sh
deleted file mode 100644
index 4a868be1..00000000
--- a/Plotting/doAllPlots.sh
+++ /dev/null
@@ -1,209 +0,0 @@
-#!/usr/bin/env bash
-
-# Authors: Katharina Eggensperger and Matthias Feurer
-# License: 3-clause BSD License
-# Contact: "automl.org"
-
-#minTimeTrace=0
-#maxTimeTrace=0
-#minErrorTrace=0
-#maxErrorTrace=0
-#minTrace=0
-#maxTrace=0
-#minBox=0
-#maxBox=0
-#specArgs=""
-
-#LDA
-#minTimeTrace=-1
-#maxTimeTrace=16
-#minErrorTrace=1260
-#maxErrorTrace=1350
-#minTrace=1000
-#maxTrace=4000
-#minBox=1260
-#maxBox=1350
-#specArgs=""
-
-#SVM
-#minTimeTrace=-1
-#maxTimeTrace=12
-#minErrorTrace=0.24
-#maxErrorTrace=0.26
-#minTrace=0.24
-#maxTrace=0.3
-#minBox=0.24
-#maxBox=0.26
-#specArgs=""
-
-#Har6
-#minTimeTrace=-1
-#maxTimeTrace=12
-#minErrorTrace=0
-#maxErrorTrace=4
-#minTrace=0
-#maxTrace=4
-#minBox=0
-#maxBox=4
-#specArgs=" -j"
-
-#Branin
-#minTimeTrace=-1
-#maxTimeTrace=12
-#minErrorTrace=0
-#maxErrorTrace=1.5
-#minTrace=0
-#maxTrace=2
-#minBox=0
-#maxBox=2
-#specArgs=" -b"
-
-
-#Net convex
-#minTimeTrace=-1
-#maxTimeTrace=12
-#minErrorTrace=0.15
-#maxErrorTrace=0.5
-#minTrace=0.15
-#maxTrace=0.5
-#minBox=0.15
-#maxBox=0.3
-#specArgs=""
-
-#Net MRBI
-minTimeTrace=-1
-maxTimeTrace=12
-minErrorTrace=0.45
-maxErrorTrace=1
-minTrace=0.45
-maxTrace=1
-minBox=0.45
-maxBox=0.7
-specArgs=""
-
-#AutoWEKA
-#minTimeTrace=-1
-#maxTimeTrace=12
-#minErrorTrace=20
-#maxErrorTrace=50
-#minTrace=20
-#maxTrace=50
-#minBox=20
-#maxBox=50
-#specArgs=""
-
-#Logistic Regression
-#minTimeTrace=0
-#maxTimeTrace=0
-#minErrorTrace=0.06
-#maxErrorTrace=0.26
-#minTrace=0.08
-#maxTrace=0.28
-#minBox=0.08
-#maxBox=0.12
-#specArgs=""
-
-echo $1
-
-smac="smac $1/smac_*_*/smac.pkl"
-tpe="tpe $1/tpe_*_*/tpe.pkl"
-spearmint="spearmint $1/spearmint_*_*/spearmint.pkl"
-random="random $1/random_*_*/random.pkl"
-
-#smac="smac $1/smac_*.pkl"
-#tpe="tpe $1/tpe_*.pkl"
-#spearmint="spearmint $1/spearmint_*.pkl"
-#random="random $1/random_*.pkl"
-
-# build arguments
-argu=""
-if [ -d "$1/smac" ]
-then
- argu="$argu $smac"
-fi
-
-if [ -d "$1/tpe" ]
-then
- argu="$argu $tpe"
-fi
-
-if [ -d "$1/spearmint" ]
-then
- argu="$argu $spearmint"
-fi
-
-if [ -d "$1/random" ]
-then
- argu="$argu $random"
-fi
-
-output="$1/Evaluation"
-mkdir $output
-echo "#########################################################################"
-echo "Generating basic statistics"
-python statistics.py $argu > "$output/statistics.txt"
-
-echo "#########################################################################"
-echo "Gather validation and test results"
-
-now=`pwd`
-cd $1
-echo `LANG=en_us_8859_1 bash "$now/results.sh" .` > "$output/results.txt"
-cd $now
-
-echo "#########################################################################"
-echo "Plotting the time trace of the optimizer"
-python plotTimeTrace.py $argu -s "$output/TimeTraceLog.png" --min $minTimeTrace --max $maxTimeTrace
-python plotTimeTrace.py $argu -s "$output/TimeTraceLog.pdf" --min $minTimeTrace --max $maxTimeTrace
-python plotTimeTrace.py $argu -l -s "$output/TimeTrace.png" --min $minTimeTrace --max $maxTimeTrace
-python plotTimeTrace.py $argu -l -s "$output/TimeTrace.pdf" --min $minTimeTrace --max $maxTimeTrace
-
-echo "#########################################################################"
-echo "Plot the error trace with standard deviation"
-python plotTraceWithStd.py $argu -a -s "$output/ErrorTraceLog.png" --min $minErrorTrace --max $maxErrorTrace $specArgs
-python plotTraceWithStd.py $argu -a -s "$output/ErrorTraceLog.pdf" --min $minErrorTrace --max $maxErrorTrace $specArgs
-python plotTraceWithStd.py $argu --nolog -a -s "$output/ErrorTrace.png" --min $minErrorTrace --max $maxErrorTrace $specArgs
-python plotTraceWithStd.py $argu --nolog -a -s "$output/ErrorTrace.pdf" --min $minErrorTrace --max $maxErrorTrace $specArgs
-
-echo "#########################################################################"
-echo "Plot the distribution of hyperparameters together with their response value"
-mkdir $output/Params
-python plot.py $tpe -s "$output/Params/TPE_para_"
-python plot.py $smac -s "$output/Params/SMAC_para_"
-python plot.py $spearmint -s "$output/Params/Spearmint_para_"
-python plot.py MIX $1/smac_*_*/smac.pkl $1/tpe_*_*/tpe.pkl $1/spearmint_*_*/spearmint.pkl -s $output/Params/MIX_para_
-
-echo "#########################################################################"
-echo "Plot an optimization trace for every single optimization"
-idx=0
-opt=""
-
-for pkl in $argu
-do
- if [ -f $pkl ]
- then
- opt=`echo $pkl | rev| cut -d"/" -f1 | rev | cut -d"." -f1`
- #python plotTrace.py $opt $pkl -s "$output/Trace_${opt}_$idx.pdf" --min $minTrace --max $maxTrace $specArgs
- #python plotTrace.py $opt $pkl -s "$output/Trace_${opt}_$idx.png" --min $minTrace --max $maxTrace $specArgs
- fi
- idx=$((idx+1))
-done
-
-for i in `seq 1000 1000 10000`
-do
- python plotTrace.py smac $1/smac_${i}_*/smac.pkl tpe $1/tpe_${i}_*/tpe.pkl spearmint $1/spearmint_${i}_*/spearmint.pkl -s "$output/Trace_all_$i.png" --min $minTrace --max $maxTrace $specArgs
- python plotTrace.py smac $1/smac_${i}_*/smac.pkl tpe $1/tpe_${i}_*/tpe.pkl spearmint $1/spearmint_${i}_*/spearmint.pkl -s "$output/Trace_all_$i.pdf" --min $minTrace --max $maxTrace $specArgs
- #python plotTrace.py smac $1/smac_${i}.pkl tpe $1/tpe_${i}.pkl spearmint $1/spearmint_${i}.pkl -s "$output/Trace_all_$i.png" --min $minTrace --max $maxTrace $specArgs
- #python plotTrace.py smac $1/smac_${i}.pkl tpe $1/tpe_${i}.pkl spearmint $1/spearmint_${i}.pkl -s "$output/Trace_all_$i.pdf" --min $minTrace --max $maxTrace $specArgs
-done
-
-echo "#########################################################################"
-echo "Plot a box-whisker plot"
-python plotBoxWhisker.py $argu -s "$output/BoxWhisker.png" --min $minBox --max $maxBox
-python plotBoxWhisker.py $argu -s "$output/BoxWhisker.pdf" --min $minBox --max $maxBox
-
-
-
-
-
-
diff --git a/Plotting/plot.py b/Plotting/plot.py
deleted file mode 100644
index 335095ce..00000000
--- a/Plotting/plot.py
+++ /dev/null
@@ -1,127 +0,0 @@
-#!/usr/bin/env python
-
-import cPickle
-import optparse
-import os
-import re
-import sys
-
-from matplotlib.colors import LogNorm
-from matplotlib.mlab import griddata
-import matplotlib.pyplot as plt
-import numpy as np
-
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__license__ = "3-clause BSD License"
-__contact__ = "automl.org"
-
-
-def translatePara(key, value):
- # sanitize all params
- newName = key
- if "LOG10" in key:
- pos = key.find("LOG10")
- newName = key[0:pos] + key[pos+5:]
- newName = newName.strip("_")
- value = np.power(10, float(value))
- elif "LOG2" in key:
- pos = key.find("LOG2")
- newName = key[0:pos] + key[pos+4:]
- newName = newName.strip("_")
- value = np.power(2, float(value))
- elif "LOG" in key:
- pos = key.find("LOG")
- newName = key[0:pos] + key[pos+3:]
- newName = newName.strip("_")
- value = np.exp(float(value))
- #Check for Q value, returns round(x/q)*q
- m = re.search('Q[0-999]{1,3}', key)
- if m != None:
- pos = newName.find(m.group(0))
- tmp = newName[0:pos] + newName[pos+3:]
- newName = tmp.strip("_")
- q = float(m.group(0)[1:])
- value = round(float(value)/q)*q
- return (newName, value)
-
-def plot_params(paramDict, save="", title=""):
- numPara = len(paramDict.keys())
- size = 10
-
- if save == "":
- # Show all paras in one plot
- f = plt.figure(figsize = (1, 5), dpi = 100)
- f.suptitle(title)
-
- plotNum = 1
-
- for para in paramDict.keys():
- plt.subplot(numPara, 1, plotNum)
- plotNum += 1
- plt.scatter(paramDict[para][:,0], paramDict[para][:,1], marker="^", \
- s=size, facecolors="none", edgecolors="k")
-
- plt.subplots_adjust(hspace=0.5)
- plt.show()
- else:
- # Make one plot for each para
- for para in paramDict.keys():
- fig, ax = plt.subplots(dpi=100)
- fig.suptitle(title + " " + para)
- ax.scatter(paramDict[para][:,0], paramDict[para][:,1], marker="^", \
- s=size, facecolors="none", edgecolors="k")
-
- save_fn = save + para
- try:
- fig.savefig(save_fn + ".png", dpi=100, facecolor='w', edgecolor='w',
- orientation='portrait', papertype=None, format=None,
- transparent=False, bbox_inches="tight", pad_inches=0.1)
- fig.savefig(save_fn + ".pdf", dpi=100, facecolor='w', edgecolor='w',
- orientation='portrait', papertype=None, format=None,
- transparent=False, bbox_inches="tight", pad_inches=0.1)
- fig.clear()
- except Exception as e:
- print e
-
-def main():
- parser = optparse.OptionParser()
- parser.add_option("-s", "--save", dest = "save", default = "", \
- help = "Where to save plot instead of showing it?")
- parser.add_option("-t", "--title", dest="title", default="TBA", \
- help = "Choose a supertitle for the plot")
- (opts, args) = parser.parse_args()
- sys.stdout.write("\nFound " + str(len(args)) + " arguments...\n")
- paramDict = dict()
- curOpt = args[0]
-
- if opts.title == "TBA":
- title = curOpt
- else:
- title = opts.title
-
- for fn in args[1:]:
- if not ".pkl" in fn:
- print "This is not a .pkl file: %s" % (fn,)
- continue
- result_file = fn
- fh = open(result_file, "r")
- trials = cPickle.load(fh)
- fh.close()
-
- keys = set()
- for i in range(len(trials["trials"])):
- for k in trials["trials"][i]["params"].keys():
- key, value = translatePara(k, trials["trials"][i]["params"][k])
- if key not in paramDict:
- paramDict[key] = list()
- else:
- paramDict[key].append((value, trials["trials"][i]["result"]))
-
- for p in paramDict.keys():
- paramDict[p] = np.array(paramDict[p])
-
- plot_params(paramDict, save=opts.save, title=title)
-
-if __name__ == "__main__":
- main()
-
diff --git a/Plotting/plotBoxWhisker.py b/Plotting/plotBoxWhisker.py
deleted file mode 100644
index 16f9a9b5..00000000
--- a/Plotting/plotBoxWhisker.py
+++ /dev/null
@@ -1,109 +0,0 @@
-#!/usr/bin/env python
-
-import cPickle
-import itertools
-import optparse
-import os
-import sys
-
-import matplotlib.gridspec as gridspec
-import matplotlib.pyplot as plt
-
-import numpy as np
-
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__license__ = "3-clause BSD License"
-__contact__ = "automl.org"
-
-
-def plot_boxWhisker(bestDict, title = "", save = "", Ymin=0, Ymax=0):
- ratio = 5
- gs = gridspec.GridSpec(ratio,1)
- fig = plt.figure(1, dpi=100)
- fig.suptitle(title)
- ax = plt.subplot(gs[0:ratio, :])
- ax.yaxis.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
-
- labels = bestDict.keys()
- bp = ax.boxplot([bestDict[i] for i in labels], 0, 'ok')
- boxlines = bp['boxes']
- for line in boxlines:
- line.set_color('k')
- line.set_linewidth(2)
-
- if Ymin != Ymax:
- ax.set_ylim([Ymin, Ymax])
- # Get medians
- medians = range(len(labels))
- for i in range(len(labels)):
- med = bp['medians'][i]
- medianX = []
- medianY = []
- for j in range(2):
- medianX.append(med.get_xdata()[j])
- medianY.append(med.get_ydata()[j])
- medians[i] = medianY[0]
- print medians
-
- # Plot xticks
- plt.xticks(range(1, len(bestDict.keys())+1), labels)
-
- # Print medians as upper labels
- top = Ymax-((Ymax-Ymin)*0.05)
- pos = np.arange(len(labels))+1
- upperLabels = [str(np.round(s, 5)) for s in medians]
- for tick,label in zip(range(len(labels)), ax.get_xticklabels()):
- k = tick % 2
- ax.text(pos[tick], top, upperLabels[tick],
- horizontalalignment='center', size='x-small')
-
- ax.set_ylabel('Minfunction evaluation')
- if save != "":
- plt.savefig(save, dpi=100, facecolor='w', edgecolor='w',
- orientation='portrait', papertype=None, format=None,
- transparent=True, bbox_inches="tight", pad_inches=0.1)
- else:
- plt.show()
-
-def main():
- parser = optparse.OptionParser()
- parser.add_option("-t", "--title", dest = "title", \
- default = "TBA", help = "Optional supertitle for plot")
- parser.add_option("--max", dest = "max", action = "store", default = 0,
- type = float, help = "Maximum of the plot")
- parser.add_option("--min", dest = "min", action = "store", default = 0,
- type = float, help = "Minimum of the plot")
- parser.add_option("-s", "--save", dest = "save", default = "", \
- help = "Where to save plot instead of showing it?")
- (opts, args) = parser.parse_args()
-
- title = ""
- if opts.title == "TBA":
- title = "Results for Optimizer(s)"
-
- sys.stdout.write("\nFound " + str(len(args)) + " arguments...")
- bestDict = dict()
- curOpt = ""
- for i in range(len(args)):
- if not ".pkl" in args[i]:
- print "Adding", args[i]
- bestDict[args[i]] = list()
- curOpt = args[i]
- continue
-
- result_file = args[i]
- fh = open(result_file, "r")
- trials = cPickle.load(fh)
- fh.close()
- bestDict[curOpt].append(min([trial['result'] for trial in trials['trials']]))
-
- plot_boxWhisker(bestDict=bestDict, title=title, save=opts.save, \
- Ymin=opts.min, Ymax=opts.max)
-
- if opts.save != "":
- sys.stdout.write("Saving plot to " + opts.save + "\n")
- else:
- sys.stdout.write("..Done\n")
-
-if __name__ == "__main__":
- main()
diff --git a/Plotting/plotBranin.py b/Plotting/plotBranin.py
deleted file mode 100644
index 96e34577..00000000
--- a/Plotting/plotBranin.py
+++ /dev/null
@@ -1,152 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-#!/usr/bin/env python
-
-import cPickle
-import itertools
-import optparse
-import os
-import sys
-
-from matplotlib.colors import LogNorm
-import matplotlib.cm as cm
-import matplotlib.gridspec as gridspec
-from matplotlib.mlab import griddata
-import matplotlib.pyplot as plt
-import numpy as np
-
-sys.path.append("../benchmarks/branin")
-import branin
-
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__contact__ = "automl.org"
-
-
-def plot_contour(trialList, nameList, save = "", title=""):
- # constraints:
- # -5 <= x <= 10, 0 <= y <= 15
- # three global optima: (-pi, 12.275), (pi, 2.275), (9.42478, 2.475), where
- # branin = 0.397887
-
- markers = itertools.cycle(['o', 's', '^', 'x'])
- colors = itertools.cycle(['b', 'g', 'r', 'k'])
- # linestyles = itertools.cycle(['-'])
- size = 5
-
- # Get handles
- ratio=5
- gs = gridspec.GridSpec(ratio,1)
- fig = plt.figure(1, dpi=100)
- fig.suptitle(title)
- ax = plt.subplot(gs[0:ratio, :])
- ax.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
- xopt = [-np.pi, np.pi, 9.42478]
- yopt = [12.275, 2.275, 2.475]
-
- # Plot Branin
- step = 0.1
- xi = np.arange(-5, 10 + step, step)
- yi = np.arange(-0, 15 + step, step)
-
- z = np.zeros([len(xi), len(yi)])
- for i in range(len(xi)):
- for j in range(len(yi)):
- #z[j, i] = np.power(np.e, branin.branin({"x":xi[i], "y":yi[j]}))
- z[j, i] = branin.branin({"x":xi[i], "y":yi[j]})
- XI, YI = np.meshgrid(xi, yi)
- cax = ax.contourf(XI, YI, z, 50, cmap=cm.gray)
- fig.colorbar(cax)
-
- # Plot Optimums after all work is done
- d = plt.scatter(xopt, yopt, marker="o", facecolor='w', edgecolor='w', s = 20*size, label="Optimum")
-
- # Get values
- for opt in range(len(nameList)):
- print nameList[opt], "has", len(trialList[opt]['trials']), "samples"
- m = markers.next()
- c = colors.next()
- x = np.zeros(len(trialList[opt]["trials"]))
- y = np.zeros(len(trialList[opt]["trials"]))
- for i in range(len(x)):
- x[i] = trialList[opt]["trials"][i]["params"]["x"]
- y[i] = trialList[opt]["trials"][i]["params"]["y"]
- a = plt.scatter(x[0:10], y[0:10], marker=m,
- s=size, facecolors=c, linewidth=0.1) # label=nameList[opt]) # + " first 10 points")
- b = plt.scatter(x[10:-10], y[10:-10], marker=m,
- linewidth=0.1, s=4*size, facecolors=c) # label="points in the middle")
- c = plt.scatter(x[-10:-1], y[-10:-1], marker=m,
- linewidth=0.1, s=6*size, facecolors=c, label=nameList[opt]) # label="last 10 points")
-
- plt.xlim([-5, 10])
- plt.ylim([-0, 15])
- plt.xlabel("X")
- plt.ylabel("Y")
- if [trial["result"] for trial in trialList[opt]["trials"]]:
- minimum = str(min([trial["result"] for trial in trialList[opt]["trials"]]))
- else:
- minimum = "NaN"
-
- # Describe the plot
- plt.title(title)
- leg = plt.legend(loc="best", fancybox=True)
- leg.get_frame().set_alpha(0.5)
-
- if save != "":
- plt.subplots_adjust(top=0.85)
- plt.savefig(save, dpi=600, facecolor='w', edgecolor='w',
- orientation='portrait', papertype=None, format=None,
- transparent=False, bbox_inches="tight", pad_inches=0.1)
- else:
- plt.show()
-
-def main():
- usage = "Usage: python plotBranin.py SMAC TPE <..> ..." + \
- "[-s ] [-t ]"
- parser = optparse.OptionParser(usage)
- parser.add_option("-s", "--save", dest = "save", default = "", \
- help = "Where to save plot instead of showing it?")
- parser.add_option("-t", "--title", dest="title", \
- default="", help="Optional supertitle for plot")
- (opts, args) = parser.parse_args()
-
- if len(args)%2 != 0:
- print "Wrong number of arguments", len(args)
- print usage
- sys.exit(1)
-
- trialList = list()
- nameList = list()
- for i in range(len(args)):
- if not ".pkl" in args[i]:
- print "Adding", args[i]
- nameList.append(args[i])
- continue
- result_file = args[i]
- fh = open(result_file, "r")
- trials = cPickle.load(fh)
- fh.close()
- trialList.append(trials)
-
- assert len(nameList) == len(trialList), ("Optimizers: %s != %s" % (len(nameList), len(trialList)))
- plot_contour(trialList=trialList, nameList=nameList, save=opts.save, title=opts.title)
-
-
-if __name__ == "__main__":
- main()
-
diff --git a/Plotting/plotTimeTrace.py b/Plotting/plotTimeTrace.py
deleted file mode 100644
index 41e9d87f..00000000
--- a/Plotting/plotTimeTrace.py
+++ /dev/null
@@ -1,224 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-#!/usr/bin/env python
-
-import cPickle
-import itertools
-import optparse
-import os
-import sys
-
-import matplotlib.pyplot as plt
-import matplotlib.gridspec as gridspec
-import numpy as np
-
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__contact__ = "automl.org"
-
-
-def plot_time_trace(timeDict, nameList, title = "", log=True, save="", verbose=False, \
- Ymax=0, Ymin=0):
- colors = itertools.cycle(['b', 'g', 'r', 'k'])
- markers = itertools.cycle(['o', 's', 'x', '^'])
- linestyles = itertools.cycle(['-'])
- size = 1
- labels = list()
- ratio = 5
- gs = gridspec.GridSpec(ratio,1)
- fig = plt.figure(figsize=(1,1), dpi=100)
-
- stdDict = dict()
- meanDict = dict()
- numDict = dict()
- minYMean = sys.maxint
- maxYMean = 0
- maxYNum = 0
- maxXNum = 0
-
- # Get mean and std for all times and optimizers
- for k in nameList:
- stdDict[k] = list()
- meanDict[k] = list()
- numDict[k] = list()
- # iterate over all funEvals
- for i in range(max([len(i) for i in timeDict[k]])):
- timeList = list()
- # check for each #funEval if it exists for this optimizer
- for run in range(len(timeDict[k])):
- if len(timeDict[k][run]) > i:
- timeList.append(timeDict[k][run][i])
- # calc mean and std
- stdDict[k].append(np.std(np.array(timeList)))
- meanDict[k].append(np.mean(np.array(timeList)))
- numDict[k].append(len(timeList))
- if min(meanDict[k]) < minYMean:
- minYMean = min(meanDict[k])
- if max(meanDict[k]) > maxYMean:
- maxYMean = max(meanDict[k])
- if max(numDict[k]) > maxYNum:
- maxYNum = max(numDict[k])
- if len(numDict[k]) > maxXNum:
- maxXNum = len(numDict[k])
-
- # Get the handles
- if verbose:
- ax1 = plt.subplot(gs[0:ratio-1, :])
-
- ax2 = plt.subplot(gs[ratio-1, :], sharex=ax1)
- ax2.set_xlabel("#Function evaluations")
- ax2.set_ylabel("#Runs with #Function evaluations")
- ax2.set_ylim([0, maxYNum+1])
- ax2.set_xlim([0, maxXNum])
- ax2.grid(True)
- fig.subplots_adjust(hspace=0.3)
- else:
- ax1 = plt.subplot(gs[0:ratio, :])
- ax1.set_xlabel("#Function evaluations")
- ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
-
- # Maybe plot on logscale
- if log and minYMean >= 0:
- ax1.set_yscale('log')
- else:
- sys.stdout.write("Plotting on logscale not possible or desired (minVal: %s)\n" %
- (minYMean,))
- log = False
-
- # Plot the stuff
- leg = list()
- for k in nameList:
- # Plot mean and std for optimizer duration
- c = colors.next()
- m = markers.next()
- x = range(len(meanDict[k]))
- l = linestyles.next()
- if not log:
- stdUp = np.array(meanDict[k])+np.array(stdDict[k])
- stdDown = np.array(meanDict[k])-np.array(stdDict[k])
- ax1.fill_between(x, meanDict[k], stdUp, \
- facecolor=c, alpha=0.3, edgecolor=c)
- ax1.fill_between(x, meanDict[k], stdDown, \
- facecolor=c, alpha=0.3, edgecolor=c)
- else:
- sys.stdout.write("INFO: Plotting std on a logscale makes no sense\n")
- ax1.plot(x, meanDict[k], color=c, linewidth=size, label=k, linestyle=l, marker=m)
- # Plot number of func evals for this experiment
- x = range(len(numDict[k]))
- if verbose:
- ax2.plot(x, numDict[k], color=c)
-
- # Descript and label the stuff
- fig.suptitle(title, fontsize=16)
- leg = ax1.legend(loc='best', fancybox=True)
- leg.get_frame().set_alpha(0.5)
- ax1.set_ylabel("Optimizer time in [sec]")
- if Ymax == Ymin:
- ax1.set_ylim([minYMean-2, maxYMean+2])
- else:
- ax1.set_ylim([Ymin, Ymax])
- ax1.set_xlim([0, maxXNum])
-
- if save != "":
- fig.set_size_inches(10,10)
- plt.savefig(save, dpi=100, facecolor='w', edgecolor='w',
- orientation='portrait', papertype=None, format=None,
- transparent=False, bbox_inches="tight", pad_inches=0.1)
- else:
- plt.show()
-
-def main():
- parser = optparse.OptionParser()
-
- # Options which are only available for this plot
- parser.add_option("-v", "--verbose", dest="verbose", action="store_true", \
- default=False, help="Add a plot, which shows #experiments")
-
- # General Options
- parser.add_option("-l", "--nolog", dest="log", action="store_true", \
- default=False, help="Do NOT plot on log scale")
- parser.add_option("--max", dest="max", action="store", default=0,
- type=float, help = "Maximum of the plot")
- parser.add_option("--min", dest="min", action="store", default=0,
- type=float, help="Minimum of the plot")
- parser.add_option("-s", "--save", dest="save", default="", \
- help="Where to save plot instead of showing it?")
- parser.add_option("-t", "--title", dest="title", default="", \
- help="Choose a supertitle for the plot")
-
-
- (opts, args) = parser.parse_args()
-
- sys.stdout.write("\nFound " + str(len(args)) + " arguments...")
- timeDict = dict()
- nameList = list()
- curOpt = ""
- for i in range(len(args)):
- if not ".pkl" in args[i]:
- print "Adding", args[i]
- timeDict[args[i]] = list()
- nameList.append(args[i])
- curOpt = args[i]
- continue
-
- result_file = args[i]
- fh = open(result_file, "r")
- trials = cPickle.load(fh)
- fh.close()
-
- # Get optimizer duration times
- timeList = list()
- # First duration
- timeList.append(trials["cv_starttime"][0] - trials["starttime"][0])
- time_idx = 0
- for i in range(len(trials["cv_starttime"][1:])):
- # Is there a next restored run?
- # if yes, does next cvstart belong to a restored run?
- if len(trials["endtime"]) > time_idx and \
- trials["cv_starttime"][i+1] > trials["endtime"][time_idx]:
- # Check whether run crashed/terminated during cv
- # Equals means that the run crashed
- if trials["cv_endtime"][i] < trials["endtime"][time_idx]:
- # No .. everything is fine
- timeList.append((trials["endtime"][time_idx] - trials["cv_endtime"][i]))
- timeList.append((trials["cv_starttime"][i + 1] - trials["starttime"][time_idx+1]))
- elif trials["endtime"][time_idx] == trials["cv_endtime"][i]:
- # Yes, but BBoM managed to set an endtime
- pass
- else:
- # Yes ... trouble
- print "Help"
- print trials["endtime"][time_idx]
- print trials["cv_endtime"][i]
- time_idx += 1
- # everything ...
- else:
- timeList.append(trials["cv_starttime"][i + 1] - trials["cv_endtime"][i])
- timeDict[curOpt].append(timeList)
-
- title = opts.title
-
- plot_time_trace(timeDict, nameList, title=title, log=not opts.log, save=opts.save, \
- verbose=opts.verbose, Ymin=opts.min, Ymax=opts.max)
- if opts.save != "":
- sys.stdout.write("Saved plot to " + opts.save + "\n")
- else:
- sys.stdout.write("..Done\n")
-
-if __name__ == "__main__":
- main()
diff --git a/Plotting/plotTrace.py b/Plotting/plotTrace.py
deleted file mode 100644
index 0542a02e..00000000
--- a/Plotting/plotTrace.py
+++ /dev/null
@@ -1,163 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-#!/usr/bin/env python
-
-import cPickle
-import itertools
-import optparse
-import os
-import sys
-
-import matplotlib.pyplot as plt
-import matplotlib.gridspec as gridspec
-import numpy as np
-import scipy.stats as sc
-
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__contact__ = "automl.org"
-
-
-def plot_optimization_trace(opts, trialList, nameList, optimum = 0, title="", Ymin=0, Ymax=0):
- markers = itertools.cycle(['o', 's', '^', 'x'])
- colors = itertools.cycle(['b', 'g', 'r', 'k'])
- linestyles = itertools.cycle(['-'])
- size = 1
- labels = list()
- plotH = list()
-
- # Get handles
- ratio=5
- gs = gridspec.GridSpec(ratio,1)
- fig = plt.figure(1, dpi=100)
- fig.suptitle(title)
- ax = plt.subplot(gs[0:ratio, :])
- ax.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
- # Get values
- minVal = sys.maxint
- maxVal = 0
- maxTrials = 0
- # This might not do what we actually want; Ideally it would only take the
- # ones that where actually run according to instance_order
- for i in range(len(nameList)):
- x = range(len(trialList[i]['trials']))
- y = np.zeros((len(trialList[i]['trials'])))
- line = np.zeros((len(trialList[i]['trials'])))
- for j, trial in enumerate(trialList[i]['trials']):
- inst_res = trial['instance_results']
- if j >= len(y):
- break
- if type(inst_res) == np.ndarray and not np.isfinite(inst_res).any():
- inst_res[np.isnan(inst_res)] = 1
- elif type(inst_res) != np.ndarray and np.isnan(inst_res):
- inst_res = 1
- tmp = sc.nanmean(np.array([inst_res, inst_res]).flat) # Black Magic
- y[j] = tmp - optimum
- line[j] = np.min(y[:j + 1])
- # Plot the stuff
- marker = markers.next()
- color = colors.next()
- l = linestyles.next()
- ax.scatter(np.argmin(line), min(line), facecolor="w", edgecolor=color, \
- s=size*10*15, marker=marker)
- ax.scatter(x, y, color=color, marker=marker, s=size*15)
- ax.plot(x, line, color=color, label=nameList[i], linestyle=l, linewidth=size)
- labels.append(trialList[i]['experiment_name'] + " (" + str(min(y)) + ")")
- if min(y) < minVal:
- minVal = min(y)
- if max(y) > maxVal:
- maxVal = max(y)
- if len(trialList[i]) > maxTrials:
- maxTrials = len(trialList[i]['trials'])
-
- # Describe and label the stuff
- ax.set_xlabel("#Function evaluations")
- ax.set_ylabel("Minfunction value")
-
- if Ymin == Ymax:
- ax.set_ylim([minVal-0.1, maxVal+0.1])
- else:
- ax.set_ylim([Ymin, Ymax])
-
- leg = ax.legend(loc='best', fancybox=True)
- leg.get_frame().set_alpha(0.5)
- ax.set_xlim([0, maxTrials])
- plt.subplots_adjust(top=0.85)
- if opts.log:
- plt.yscale('log')
- if opts.save != "":
- plt.savefig(opts.save, dpi=100, facecolor='w', edgecolor='w',
- orientation='portrait', papertype=None, format=None,
- transparent=False, bbox_inches="tight", pad_inches=0.1)
- else:
- plt.show()
-
-def main():
- parser = optparse.OptionParser()
- # Options for specific functions
- parser.add_option("-o", "--optimum", type = float, dest = "optimum",
- default = 0, help = "Optimum")
- parser.add_option("-b", "--branin", dest = "branin", action = "store_true",
- default = False, help = "Automatic shift for branin function")
- parser.add_option("-c", "--camel", dest="camelback", action="store_true",
- default = False, help="Automatic shift for camelback function")
- parser.add_option("-i", "--har3", dest = "har3", action = "store_true",
- default = False, help = "Automatic shift for hartmann3 function")
- parser.add_option("-j", "--har6", dest = "har6", action = "store_true",
- default = False, help = "Automatic shift for hartmann6 function")
- parser.add_option("-g", "--gold", dest = "gold", action = "store_true",
- default = False, help = "Automatic shift for goldstein function")
-
- # General Options
- parser.add_option("-l", "--log", dest = "log", action = "store_true",
- default = False, help = "Plot on the log scale")
- parser.add_option("--max", dest = "max", action = "store", default = 0,
- type = float, help = "Maximum of the plot")
- parser.add_option("--min", dest = "min", action = "store", default = 0,
- type = float, help = "Minimum of the plot")
- parser.add_option("-s", "--save", dest = "save", default = "", \
- help = "Where to save plot instead of showing it?")
- parser.add_option("-t", "--title", dest="title", \
- default="", help="Optional supertitle for plot")
- (opts, args) = parser.parse_args()
-
- optimum = opts.optimum
- if opts.branin: optimum = 0.397887
- if opts.camelback: optimum = -1.031628453
- if opts.har3: optimum = -3.86278
- if opts.har6: optimum = -3.32237
- if opts.gold: optimum = -3
-
- sys.stdout.write("Found " + str(len(args)) + " pkl file(s)\n")
- trialList = list()
- nameList = list()
- for i in range(len(args)):
- if not ".pkl" in args[i]:
- print "Adding", args[i]
- nameList.append(args[i])
- continue
- result_file = args[i]
- fh = open(result_file, "r")
- trials = cPickle.load(fh)
- fh.close()
- trialList.append(trials)
- sys.stdout.write("Plotting trace\n")
- plot_optimization_trace(opts, trialList, nameList, optimum, title=opts.title, Ymax=opts.max, Ymin=opts.min)
-
-if __name__ == "__main__":
- main()
diff --git a/Plotting/plotTraceWithStd.py b/Plotting/plotTraceWithStd.py
deleted file mode 100644
index e7f54711..00000000
--- a/Plotting/plotTraceWithStd.py
+++ /dev/null
@@ -1,236 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-#!/usr/bin/env python
-
-import cPickle
-import itertools
-import optparse
-import os
-import sys
-
-import matplotlib.pyplot as plt
-import matplotlib.gridspec as gridspec
-import numpy as np
-
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__contact__ = "automl.org"
-
-
-def plot_optimization_trace(trialList, nameList, optimum=0, title="", \
- log=True, save="", Ymax=0, Ymin=0):
- markers = itertools.cycle(['o', 's', 'x', '^'])
- colors = itertools.cycle(['b', 'g', 'r', 'k'])
- linestyles = itertools.cycle(['-'])
- size = 1
- labels = list()
- plotH = list()
-
- ratio = 5
- gs = gridspec.GridSpec(ratio,1)
- fig = plt.figure(1, dpi=100)
- fig.suptitle(title, fontsize=16)
- ax1 = plt.subplot(gs[0:ratio, :])
- ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
- minVal = sys.maxint
- maxVal = 0
- maxTrials = 0
-
- trialListMeans = list()
- trialListStd = list()
- trialListMax = list()
- trialListMin = list()
-
- # One trialList represents all runs from one optimizer
- for i in range(len(trialList)):
- trialListMeans.append(np.mean(np.array(trialList[i]), axis=0))
- trialListStd.append(np.std(np.array(trialList[i]), axis=0))
- trialListMax.append(np.max(np.array(trialList[i]), axis=0))
- trialListMin.append(np.min(np.array(trialList[i]), axis=0))
-
- fig.suptitle(title, fontsize=16)
-
- # Plot the average error and std
- for i in range(len(trialListMeans)):
- x = range(1,len(trialListMeans[i])+1)
- y = trialListMeans[i] - optimum
- m = markers.next()
- c = colors.next()
- l = linestyles.next()
- if log == False:
- stdUp = y+trialListStd[i]
- stdDown = y-trialListStd[i]
- ax1.fill_between(x, y, stdUp, \
- facecolor=c, alpha=0.3, edgecolor=c)
- ax1.fill_between(x, y, stdDown, \
- facecolor=c, alpha=0.3, edgecolor=c)
- #ax1.errorbar(x, y, yerr=trialListStd[i], color='k')
- else:
- sys.stdout.write("INFO: Plotting std on a logscale makes no sense\n")
- ax1.plot(x, y, color=c, linewidth=size, label=nameList[i][0], \
- linestyle=l, marker=m)
-
- if min(y) < minVal:
- minVal = min(y)
- if max(y) > maxVal:
- maxVal = max(y)
- if len(trialListMeans[i]) > maxTrials:
- maxTrials = len(trialListMeans[i])
-
- # Maybe plot on logscale
- if float(minVal) > 0.0 and log:
- ax1.set_yscale('log')
- else:
- sys.stdout.write("Plotting on logscale not possible or desired (minVal: %s)\n" %
- (minVal,))
- log = False
-
- # Descript and label the stuff
- leg = ax1.legend(loc='best', fancybox=True)
- leg.get_frame().set_alpha(0.5)
- ax1.set_xlabel("#Function evaluations")
- ax1.set_ylabel("Minfunction value")
- if Ymax == Ymin:
- ax1.set_ylim([minVal-0.1, maxVal+1])
- else:
- ax1.set_ylim([Ymin, Ymax])
- ax1.set_xlim([0, 100])# maxTrials])
-
- plt.tight_layout()
- plt.subplots_adjust(top=0.85)
- if save != "":
- plt.savefig(save, dpi=100, facecolor='w', edgecolor='w',
- orientation='portrait', papertype=None, format=None,
- transparent=False, bbox_inches="tight", pad_inches=0.1)
- else:
- plt.show()
-
-def main():
- parser = optparse.OptionParser()
- # Options for specific functions
- parser.add_option("-b", "--branin", dest="branin", action="store_true",
- default = False, help="Automatic shift for branin function")
- parser.add_option("-c", "--camel", dest="camelback", action="store_true",
- default = False, help="Automatic shift for camelback function")
- parser.add_option("-i", "--har3", dest="har3", action="store_true",
- default = False, help="Automatic shift for hartmann3 function")
- parser.add_option("-j", "--har6", dest="har6", action="store_true",
- default = False, help="Automatic shift for hartmann6 function")
- parser.add_option("-g", "--gold", dest="gold", action="store_true",
- default = False, help="Automatic shift for goldstein function")
- parser.add_option("-o", "--optimum", type=float, dest="optimum",
- default=0, help="Optimum")
-
- # Options which are available only for this plot
- parser.add_option("-a", "--autofill", dest="autofill", default=False, \
- action="store_true", help="Fill trace automatically")
- parser.add_option("--fill", dest="fill", type=int, default=0,
- help="In case that a trace is not long enough, fill it with the" \
- + " best result so far until there are entries")
-
- # General Options
- parser.add_option("-l", "--nolog", dest="log", action="store_true", \
- default=False, help="Do NOT plot on log scale")
- parser.add_option("--max", dest="max", action="store", default=0,
- type=float, help="Maximum of the plot")
- parser.add_option("--min", dest="min", action="store", default=0,
- type=float, help="Minimum of the plot")
- parser.add_option("-s", "--save", dest = "save", default="", \
- help = "Where to save plot instead of showing it?")
- parser.add_option("-t", "--title", dest="title", \
- default="", help="Optional supertitle for plot")
-
- (opts, args) = parser.parse_args()
-
- sys.stdout.write("Generate plot for ")
- optimum = opts.optimum
- title = opts.title
- if opts.branin:
- optimum = 0.397887
- sys.stdout.write("branin")
- if opts.camelback:
- optimum = -1.031628453
- sys.stdout.write("camelback")
- if opts.har3:
- optimum = -3.86278
- sys.stdout.write("hartmann3")
- if opts.har6:
- optimum = -3.32237
- sys.stdout.write("hartmann6")
- if opts.gold:
- optimum = 3
- sys.stdout.write("goldstein")
-
- sys.stdout.write("\nFound " + str(len(args)) + " arguments...")
- nameList = list()
- trialList = list()
- lenList = list()
- for i in range(len(args)):
- if not ".pkl" in args[i]:
- print "Adding", args[i]
- nameList.append([args[i],0])
- trialList.append(list())
- lenList.append(0)
- continue
- else:
- nameList[-1][1] += 1
-
- result_file = args[i]
- fh = open(result_file, "r")
- trials = cPickle.load(fh)
- fh.close()
-
- trace = list()
- currentbest = np.nanmax(np.array([trial["result"] for trial in trials['trials']]))
-
- for result in [trial["result"] for trial in trials['trials']]:
- if result < currentbest:
- currentbest = result
- trace.append(currentbest)
- """
- # Make sure that trace is long enough
- if opts.fill != 0 and opts.fill > len(trace):
- print "Pickle file", args[i], "does only contain", len(trace), "entries"
- trace.extend([currentbest for i in range(opts.fill - len(trace))])
- print len(trace)
- """
- if len(trace) > lenList[-1]:
- lenList[-1] = len(trace)
- trialList[-1].append(np.array(trace))
-
- if opts.autofill:
- assert(len(trialList) == len(lenList))
- print lenList
- for i in range(len(trialList)):
- for t in range(len(trialList[i])):
- if len(trialList[i][t]) < lenList[i]:
- diff = lenList[i] - len(trialList[i][t])
- trialList[i][t] = np.append(trialList[i][t], [trialList[i][t][-1] for x in range(diff)])
- assert len(trialList[i][t]) == lenList[i], \
- "%s, %s" % (len(trialList[i][t]), lenList[i])
-
- plot_optimization_trace(trialList, nameList, optimum, title=title, \
- log=not opts.log, save=opts.save, Ymin=opts.min, Ymax=opts.max)
-
- if opts.save != "":
- sys.stdout.write("Saved plot to " + opts.save + "\n")
- else:
- sys.stdout.write("..Done\n")
-
-if __name__ == "__main__":
- main()
diff --git a/Plotting/plothar6.py b/Plotting/plothar6.py
deleted file mode 100644
index d5cf2cdd..00000000
--- a/Plotting/plothar6.py
+++ /dev/null
@@ -1,245 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-#!/usr/bin/env python
-
-import cPickle
-import optparse
-import os
-import sys
-
-from matplotlib.colors import LogNorm
-from matplotlib.mlab import griddata
-import matplotlib.pyplot as plt
-import numpy as np
-import math
-
-sys.path.append("../")
-import functions.har6 as har6
-
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__contact__ = "automl.org"
-
-
-def plot_contour(trials, save="", title=""):
- # 6d Hartmann function
- # constraints:
- # 0 <= xi <= 1, i = 1..6
- # global optimum at (0.20169, 0.150011, 0.476874, 0.275332, 0.311652, 0.6573),
- # where har6 = -3.32237
-
- xopt = 0.20169
- yopt = 0.150011
- zopt = 0.476874
- aopt = 0.275332
- bopt = 0.311652
- copt = 0.6573
- optimum = -3.32237
-
- x = np.zeros(len(trials["trials"]))
- y = np.zeros(len(trials["trials"]))
- z = np.zeros(len(trials["trials"]))
- a = np.zeros(len(trials["trials"]))
- b = np.zeros(len(trials["trials"]))
- c = np.zeros(len(trials["trials"]))
- result = np.zeros(len(trials["params"]))
- for i in range(len(x)):
- x[i] = trials["trials"][i]["params"]["x"]
- y[i] = trials["trials"][i]["params"]["y"]
- z[i] = trials["trials"][i]["params"]["z"]
- a[i] = trials["trials"][i]["params"]["a"]
- b[i] = trials["trials"][i]["params"]["b"]
- c[i] = trials["trials"][i]["params"]["c"]
- result[i] = trials['results'][i] - optimum
- if math.isnan(x[i]) or math.isnan(y[i]) or math.isnan(result[i]):
- print "#################sdfkdfs!"
- maxRes = max(result)
- minRes = min(result)
-
- f = plt.figure(1, figsize = (10, 10), dpi = 100)
-
- # Find minimum to generate title
- if trials["results"] != []:
- minimum = str(min(trials["results"]))
- else:
- minimum = "NaN"
- f.suptitle(title + \
- " with Hartmann6d\nUsed " + str(len(x)) + " trials, " \
- + "best found value: " + minimum + " (-3.32237)", fontsize=16)
-
- plt.subplot(611)
- size = 10
- aplt = plt.scatter(x[0:10], result[0:10], marker="^", \
- s=size, facecolors="none", edgecolors="k", label="first 10 points")
- bplt = plt.scatter(x[10:-10], result[10:-10], marker="o", \
- s=4*size, facecolors="none", edgecolors="k", label="points in the middle")
- cplt = plt.scatter(x[-10:-1], result[-10:-1], marker="*", \
- s=4*4*size, facecolors="none", edgecolors="k", label="last 10 points")
- plt.plot([xopt, xopt], [0, maxRes], color="r")
- plt.xlim([0, 1])
- plt.ylim([0, maxRes])
- plt.xlabel("X")
- plt.ylabel("Result - Optimum")
- #plt.yscale('log')
- leg = plt.legend((aplt, bplt, cplt), (aplt.get_label(), bplt.get_label(), \
- cplt.get_label()), fancybox=True, loc="best")
- leg.get_frame().set_alpha(0.5)
-
- plt.subplot(612)
- aplt = plt.scatter(y[0:10], result[0:10], marker="^", \
- s=size, facecolors="none", edgecolors="k", label="first 10 points")
- bplt = plt.scatter(y[10:-10], result[10:-10], marker="o", \
- s=4*size, facecolors="none", edgecolors="k", label="points in the middle")
- cplt = plt.scatter(y[-10:-1], result[-10:-1], marker="*", \
- s=4*4*size, facecolors="none", edgecolors="k", label="last 10 points")
- plt.plot([yopt, yopt], [0, maxRes], color="r")
- plt.xlim([0, 1])
- plt.ylim([0, maxRes])
- plt.xlabel("Y")
- plt.ylabel("Result - Optimum")
- #plt.yscale('log')
- leg = plt.legend((aplt, bplt, cplt), (aplt.get_label(), bplt.get_label(), \
- cplt.get_label()), fancybox=True, loc="best")
- leg.get_frame().set_alpha(0.5)
-
- plt.subplot(613)
- aplt = plt.scatter(z[0:10], result[0:10], marker="^", \
- s=size, facecolors="none", edgecolors="k", label="first 10 points")
- bplt = plt.scatter(z[10:-10], result[10:-10], marker="o", \
- s=4*size, facecolors="none", edgecolors="k", label="points in the middle")
- cplt = plt.scatter(z[-10:-1], result[-10:-1], marker="*", \
- s=4*4*size, facecolors="none", edgecolors="k", label="last 10 points")
- plt.plot([zopt, zopt], [0, maxRes], color="r")
-
- plt.xlim([0, 1])
- plt.ylim([0, maxRes])
- plt.xlabel("Z")
- plt.ylabel("Result - Optimum")
- #plt.yscale('log')
- leg = plt.legend((aplt, bplt, cplt), (aplt.get_label(), bplt.get_label(), \
- cplt.get_label()), fancybox=True, loc="best")
- leg.get_frame().set_alpha(0.5)
-
- plt.subplot(614)
- aplt = plt.scatter(a[0:10], result[0:10], marker="^", \
- s=size, facecolors="none", edgecolors="k", label="first 10 points")
- bplt = plt.scatter(a[10:-10], result[10:-10], marker="o", \
- s=4*size, facecolors="none", edgecolors="k", label="points in the middle")
- cplt = plt.scatter(a[-10:-1], result[-10:-1], marker="*", \
- s=4*4*size, facecolors="none", edgecolors="k", label="last 10 points")
- plt.plot([zopt, zopt], [0, maxRes], color="r")
-
- plt.xlim([0, 1])
- plt.ylim([0, maxRes])
- plt.xlabel("A")
- plt.ylabel("Result - Optimum")
- #plt.yscale('log')
- leg = plt.legend((aplt, bplt, cplt), (aplt.get_label(), bplt.get_label(), \
- cplt.get_label()), fancybox=True, loc="best")
- leg.get_frame().set_alpha(0.5)
-
-
- plt.subplot(615)
- aplt = plt.scatter(b[0:10], result[0:10], marker="^", \
- s=size, facecolors="none", edgecolors="k", label="first 10 points")
- bplt = plt.scatter(b[10:-10], result[10:-10], marker="o", \
- s=4*size, facecolors="none", edgecolors="k", label="points in the middle")
- cplt = plt.scatter(b[-10:-1], result[-10:-1], marker="*", \
- s=4*4*size, facecolors="none", edgecolors="k", label="last 10 points")
- plt.plot([zopt, zopt], [0, maxRes], color="r")
-
- plt.xlim([0, 1])
- plt.ylim([0, maxRes])
- plt.xlabel("B")
- plt.ylabel("Result - Optimum")
- #plt.yscale('log')
- leg = plt.legend((aplt, bplt, cplt), (aplt.get_label(), bplt.get_label(), \
- cplt.get_label()), fancybox=True, loc="best")
- leg.get_frame().set_alpha(0.5)
-
-
- plt.subplot(616)
- aplt = plt.scatter(c[0:10], result[0:10], marker="^", \
- s=size, facecolors="none", edgecolors="k", label="first 10 points")
- bplt = plt.scatter(c[10:-10], result[10:-10], marker="o", \
- s=4*size, facecolors="none", edgecolors="k", label="points in the middle")
- cplt = plt.scatter(c[-10:-1], result[-10:-1], marker="*", \
- s=4*4*size, facecolors="none", edgecolors="k", label="last 10 points")
- plt.plot([zopt, zopt], [0, maxRes], color="r")
-
- plt.xlim([0, 1])
- plt.ylim([0, maxRes])
- plt.xlabel("C")
- plt.ylabel("Result - Optimum")
- #plt.yscale('log')
- leg = plt.legend((aplt, bplt, cplt), (aplt.get_label(), bplt.get_label(), \
- cplt.get_label()), fancybox=True, loc="best")
- leg.get_frame().set_alpha(0.5)
-
- plt.tight_layout()
- plt.subplots_adjust(top=0.85)
- if save != "":
- plt.savefig(save, dpi=100, facecolor='w', edgecolor='w',
- orientation='portrait', papertype=None, format=None,
- transparent=False, bbox_inches="tight", pad_inches=0.1)
- else:
- plt.show()
- f.clear()
-
-def main():
- parser = optparse.OptionParser()
- parser.add_option("-s", "--save", dest = "save", default = "", \
- help = "Where to save plot instead of showing it?")
- (opts, args) = parser.parse_args()
-
- if opts.save != "":
- sys.stdout.write("Creating %s plots\n" % (len(args),) )
- for idx,i in enumerate(args):
- result_file = i
- print i
- if os.path.exists(result_file) != True:
- sys.stderr.write("No valid file: %s\n" % (result_file,) )
- fh = open(result_file, "r")
- trials = cPickle.load(fh)
- fh.close()
- idxx = result_file.find(trials["experiment_name"])
- title = result_file[idxx:-(len(trials["experiment_name"])+5)]
- plot_contour(trials, save=opts.save + title, title=title)
- print len(trials['results'])
- else:
- result_file = args[0]
- fh = open(result_file, "r")
- trials = cPickle.load(fh)
- fh.close()
-
- """
- for i in range(10):
- print "x:" + str(trials["params"][i]["x"]) + \
- " y:" + str(trials["params"][i]["y"]) + \
- " z:" + str(trials["params"][i]["z"]) + \
- " a:" + str(trials["params"][i]["a"]) + \
- " b:" + str(trials["params"][i]["b"]) + \
- " c:" + str(trials["params"][i]["c"]) + \
- " res:" + str(trials["results"][i])
- """
- idxx = result_file.find(trials["experiment_name"])
- title = result_file[idxx:-(len(trials["experiment_name"])+5)]
- plot_contour(trials, title=title)
-
-if __name__ == "__main__":
- main()
diff --git a/Plotting/results.sh b/Plotting/results.sh
deleted file mode 100644
index 3e93f392..00000000
--- a/Plotting/results.sh
+++ /dev/null
@@ -1,115 +0,0 @@
-#!/bin/sh
-
-echo "Looking for SMAC:"
-
-printf "%5s | %4s(%5s) | %5s | %10s | %10s\n" "Seed" "#run" "crash" "#iter" "Performance" "Test-Perf"
-for i in `seq 1000 1000 10000`
-do
- directory=`ls | grep "^smac_${i}_"`
- if [ -f "${directory}/smac.out" ]
- then
- it=`cat ${directory}/smac.out | grep "Model/Iteration used:" | tail -1`
- it=`echo $it | cut -d' ' -f3`
-
- per=`cat ${directory}/smac.out | grep "Performance of the Incumbent:" | tail -1`
- per=`echo $per | cut -d' ' -f5`
-
- num=`cat ${directory}/smac.out | grep "Algorithm Runs used:" | tail -1`
- num=`echo $num | cut -d' ' -f4`
-
- numC=`ls ${directory}/ | grep 'instance.out$' | wc -l`
- numC=$(($numC - 1))
-
- test_error=999
- test_file=`ls "$directory/" | grep "_test_run.out"`
- if [ -f "${directory}/$test_file" ]
- then
- test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1`
- test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1`
- fi
- printf "%5s | %4s(%5s) | %5s | %10f | %10f\n" "$i" "$num" "$numC" "$it" "$per" "$test_error"
- fi
-
-done
-
-echo "Looking for TPE:"
-
-printf "%5s | %4s(%5s) | %10s | %10s\n" "Seed" "#run" "crash" "Performance" "Test-Perf"
-for i in `seq 1000 1000 10000`
-do
- directory=`ls | grep "^tpe_${i}_"`
- if [ -a "${directory}/tpe.out" ]
- then
- num=`cat ${directory}/tpe.out | grep "Result:" | wc -l`
-
- per=`cat ${directory}/tpe.out | grep "Result:" | sort -r | tail -1`
- per=`echo $per | cut -d' ' -f2`
-
- numC=`ls ${directory}/ | grep 'instance.out$' | wc -l`
- numC=$(($numC - 1))
-
- test_error=999
- test_file=`ls -1 "$directory" | grep "_test_run.out"`
- if [ -f "$directory/$test_file" ]
- then
- test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1`
- test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1`
- fi
- printf "%5s | %4s(%5s) | %10f | %10f\n" "$i" "$num" "$numC" "$per" "$test_error"
- fi
-done
-
-echo "Looking for Spearmint:"
-
-printf "%5s | %4s(%5s) | %10s | %10s\n" "Seed" "#run" "crash" "Performance" "Test-Perf"
-for i in `seq 1000 1000 10000`
-do
- directory=`ls | grep "^spearmint_${i}_"`
- if [ -a "${directory}/spearmint.out" ]
- then
- num=`cat ${directory}/spearmint.out | grep " pending " | tail -1`
- num=`echo $num | cut -d' ' -f5`
- per=`cat ${directory}/spearmint.out | grep "best:" | tail -1`
- per=`echo $per | cut -d' ' -f3`
-
- numC=`ls ${directory}/ | grep 'instance.out$' | wc -l`
- numC=$(($numC - 1))
-
- test_error=999
- test_file=`ls -1 "$directory" | grep "_test_run.out"`
- if [ -f "$directory/$test_file" ]
- then
- test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1`
- test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1`
- fi
- printf "%5s | %4s(%5s) | %10f | %10f\n" "$i" "$num" "$numC" "$per" "$test_error"
- fi
-done
-
-echo "Looking for Random:"
-
-printf "%5s | %4s(%5s) | %10s | %10s\n" "Seed" "#run" "crash" "Performance" "Test-Perf"
-for i in `seq 1000 1000 10000`
-do
- directory=`ls | grep "^randomtpe_${i}_"`
- if [ -a "${directory}/randomtpe.out" ]
- then
- num=`cat ${directory}/randomtpe.out | grep "Result:" | wc -l`
-
- per=`cat ${directory}/randomtpe.out | grep "Result:" | sort -r | tail -1`
- per=`echo $per | cut -d' ' -f2`
-
- numC=`ls ${directory}/ | grep 'instance.out$' | wc -l`
- numC=$(($numC - 1))
-
- test_error=999
- test_file=`ls -1 "$directory" | grep "_test_run.out"`
- if [ -f "$directory/$test_file" ]
- then
- test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1`
- test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1`
- fi
- printf "%5s | %4s(%5s) | %10f | %10f\n" "$i" "$num" "$numC" "$per" "$test_error"
- fi
-done
-
diff --git a/Plotting/statistics.py b/Plotting/statistics.py
deleted file mode 100644
index 47a05a18..00000000
--- a/Plotting/statistics.py
+++ /dev/null
@@ -1,209 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-#!/usr/bin/env python
-
-import cPickle
-import optparse
-import sys
-
-import numpy
-import numpy.ma as ma
-from scipy import stats
-
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__contact__ = "automl.org"
-
-
-def _getBestTrial(filename, cut=None):
- try:
- fh = open(filename, "r")
- trials = cPickle.load(fh)
- fh.close()
-
- current_best = numpy.Inf
- best_idx = 0
- if cut is None:
- cut = len(trials['trials'])
- print filename, "#Trials", len(trials['trials'])
- for i, trial in enumerate(trials['trials'][:cut]):
- result = trial['result']
- if result < current_best:
- best_idx = i
- current_best = result
- if current_best == numpy.Inf:
- raise Exception("%s does not contain any results" % filename)
- return current_best, best_idx
- except Exception as e:
- print "Problem with ", filename, e
- sys.stdout.flush()
- return None, None
-
-
-def _mann_whitney_u(x, y=None):
- """
- Calculate the Mann-Whitney-U test.
-
- The Wilcoxon signed-rank test tests the null hypothesis that two related paired
- samples come from the same distribution. In particular, it tests whether the
- distribution of the differences x - y is symmetric about zero.
- It is a non-parametric version of the paired T-test.
- """
- # A significance-dict for a two-tailed test with 0.05 confidence
- # from http://de.wikipedia.org/wiki/Wilcoxon-Mann-Whitney-Test
- significance_table = \
- [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
- [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2],
- [0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8],
- [0, 0, 0, 0, 1, 2, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 11, 12, 13, 14],
- [0, 0, 0, 0, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13, 14, 15, 17, 18, 19, 20],
- [0, 0, 0, 0, 0, 5, 6, 8, 10, 11, 13, 14, 16, 17, 19, 21, 22, 24, 25, 27],
- [0, 0, 0, 0, 0, 0, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34],
- [0, 0, 0, 0, 0, 0, 0, 13, 15, 17, 19, 22, 24, 26, 29, 31, 34, 36, 38, 41],
- [0, 0, 0, 0, 0, 0, 0, 0, 17, 20, 23, 26, 28, 31, 34, 37, 39, 42, 45, 48],
- [0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 26, 29, 33, 36, 39, 42, 45, 48, 52, 55],
- [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 33, 37, 40, 44, 47, 51, 55, 58, 62],
- [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 41, 45, 49, 53, 57, 61, 65, 69],
- [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 50, 54, 59, 63, 67, 72, 76],
- [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 59, 64, 69, 74, 78, 83],
- [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 70, 75, 80, 85, 90],
- [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 81, 86, 92, 98],
- [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 93, 99, 105],
- [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 106, 112],
- [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 119],
- [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127]]
-
- U, p = stats.mannwhitneyu(x, y)
-
- x = ma.asarray(x).compressed().view(numpy.ndarray)
- y = ma.asarray(y).compressed().view(numpy.ndarray)
-
- n1 = len(x)
- n2 = len(y)
- print n1, n2
- if n1 > n2:
- tmp = n2
- n2 = n1
- n1 = tmp
-
- if n1 < 5 or n2 < 5:
- return 10000, 10000
- if n1 < 20 or n2 < 20:
- print "WARNING: scipy.stat might not be accurate, p value is %f and significance according to table is %s" % \
- (p, U <= significance_table[n1 - 1][n2 - 1])
- return U, p
-
-def _unpaired_ttest(x, y, equal_var=True):
- U, p = stats.ttest_ind(x, y, equal_var=equal_var)
- return U, p
-
-def main():
- usage = "python statistics.py SMAC smac_*/smac.pkl TPE tpe_*/tpe.pkl\n"
- parser = optparse.OptionParser(usage)
- parser.add_option("-c", "--c", type=int, dest="cut",
- default=None,
- help="Only consider that many evaluations")
- (opts, args) = parser.parse_args()
-
- bestDict = dict()
- idxDict = dict()
- keys = list()
- curOpt = ""
- for i in range(len(args)):
- if not ".pkl" in args[i]:
- bestDict[args[i]] = list()
- idxDict[args[i]] = list()
- keys.append(args[i])
- curOpt = args[i]
- continue
- best, idx =_getBestTrial(args[i], opts.cut)
- if best is None and idx is None:
- continue
-
- # best = best * 100
-
- bestDict[curOpt].append(best)
- idxDict[curOpt].append(idx)
-
- for k in keys:
- sys.stdout.write("%10s: %s experiments\n" % (k, len(bestDict[k])))
- """
- sys.stdout.write("Wilcoxon Test--------------------------------------------------------\n")
- for idx, k in enumerate(keys):
- if len(keys) > 1:
- for jdx, j in enumerate(keys[idx+1:]):
- #if len(bestDict[k]) == len(bestDict[j]):
- T, p = _mann_whitney_u(bestDict[k], bestDict[j])
- sys.stdout.write("%10s vs %10s" % (k,j))
- sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" %
- (T, p, p*100))
- #else:
- # print "WARNING: %s has not the same number of function evals than %s" % \
- # (k, j)
- # continue
- else:
- T, p = _mann_whitney_u(bestDict[k], bestDict[k])
- sys.stdout.write("%10s vs. %10s" % (k,k))
- sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" %
- (T, p, p*100))
- """
- sys.stdout.write("Unpaired t-tests-----------------------------------------------------\n")
- for idx, k in enumerate(keys):
- if len(keys) > 1:
- for jdx, j in enumerate(keys[idx+1:]):
- #if len(bestDict[k]) == len(bestDict[j]):
- T_t, p_t = _unpaired_ttest(bestDict[k], bestDict[j], equal_var=True)
- T_f, p_f = _unpaired_ttest(bestDict[k], bestDict[j], equal_var=False)
- T_rt, p_rt = _unpaired_ttest(numpy.round(bestDict[k], 3), numpy.round(bestDict[j], 3), equal_var=True)
- T_rf, p_rf = _unpaired_ttest(numpy.round(bestDict[k], 3), numpy.round(bestDict[j], 3), equal_var=False)
- sys.stdout.write("Standard independent 2 sample test, equal population variance\n")
- sys.stdout.write("%10s vs %10s" % (k,j))
- sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%) \n" %
- (T_t, p_t, p_t*100))
- sys.stdout.write("Rounded: ")
- sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" %
- (T_rt, p_rt, p_rt*100))
- sys.stdout.write("Welch's t-test, no equal population variance\n")
- sys.stdout.write("%10s vs %10s" % (k,j))
- sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" %
- (T_f, p_f, p_f*100))
- sys.stdout.write("Rounded: ")
- sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" %
- (T_rf, p_rf, p_rf*100))
- sys.stdout.write("Best Value-----------------------------------------------------------\n")
- for k in keys:
- stdBest = numpy.std(bestDict[k])
- minBest = numpy.min(bestDict[k])
- maxBest = numpy.max(bestDict[k])
- meanBest = numpy.mean(bestDict[k])
- sys.stdout.write("%10s: %10.5f (min: %10.5f, max: %10.5f, std: %5.3f)\n" %
- (k, meanBest, minBest, maxBest, stdBest))
-
- sys.stdout.write("Needed Trials--------------------------------------------------------\n")
- for k in keys:
- stdIdx = numpy.std(idxDict[k])
- meanIdx = numpy.mean(idxDict[k])
- minIdx = numpy.min(idxDict[k])
- maxIdx = numpy.max(idxDict[k])
- sys.stdout.write("%10s: %10.5f (min: %10.5f, max: %10.5f, std: %5.3f)\n" %
- (k, meanIdx, minIdx, maxIdx, stdIdx))
-
- sys.stdout.write("------------------------------------------------------------------------\n")
-
-if __name__ == "__main__":
- main()
diff --git a/__init__.py b/benchmarks/__init__.py
similarity index 100%
rename from __init__.py
rename to benchmarks/__init__.py
diff --git a/benchmarks/branin/__init__.py b/benchmarks/branin/__init__.py
new file mode 100644
index 00000000..24cd9aaa
--- /dev/null
+++ b/benchmarks/branin/__init__.py
@@ -0,0 +1,2 @@
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
\ No newline at end of file
diff --git a/benchmarks/branin/branin.py b/benchmarks/branin/branin.py
index 46deb67c..304c1851 100644
--- a/benchmarks/branin/branin.py
+++ b/benchmarks/branin/branin.py
@@ -16,46 +16,26 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-import sys
-from numpy import NaN, pi, cos, ndarray
-import benchmark_util
-
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__contact__ = "automl.org"
+import HPOlib.benchmark_util as benchmark_util
+import HPOlib.benchmark_functions as benchmark_functions
-def branin(params, **kwargs):
-# branin function
-# The number of variables n = 2.
-# constraints:
-# -5 <= x <= 10, 0 <= y <= 15
-# three global optima: (-pi, 12.275), (pi, 2.275), (9.42478, 2.475), where
-# branin = 0.397887
- if "x" not in params or "y" not in params:
- raise ValueError("No params found ['x', 'y']\n")
- x = float(params["x"])
- y = float(params["y"])
- if type(x) == ndarray or type(y) == ndarray:
- x = x[0]
- y = y[0]
+import time
- if -5 > x or x > 10:
- raise ValueError("X value not in between -5 and 10")
- if 0 > y or y > 15:
- raise ValueError("Y value not in between 0 and 15")
-
- result = (y-(5.1/(4*pi**2))*x**2+5*x/pi-6)**2
- result += 10*(1-1/(8*pi))*cos(x)+10
- return result
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
def main(params, **kwargs):
print 'Params: ', params,
- y = branin(params, **kwargs)
+ y = benchmark_functions.save_branin(params, **kwargs)
print 'Result: ', y
return y
if __name__ == "__main__":
+ starttime = time.time()
args, params = benchmark_util.parse_cli()
result = main(params, **args)
- print "Result", result
\ No newline at end of file
+ duration = time.time() - starttime
+ print "Result for ParamILS: %s, %f, 1, %f, %d, %s" % \
+ ("SAT", abs(duration), result, -1, str(__file__))
diff --git a/benchmarks/branin/config.cfg b/benchmarks/branin/config.cfg
index 85f55f8f..4a7dbc05 100644
--- a/benchmarks/branin/config.cfg
+++ b/benchmarks/branin/config.cfg
@@ -1,7 +1,13 @@
[SMAC]
-p = smac/params.txt
+p = smac_2_06_01-dev/params.pcs
-[DEFAULT]
-function = branin.py
-numberOfJobs = 200
-result_on_terminate = 1
+[TPE]
+space = space.py
+
+[SPEARMINT]
+config = config.pb
+
+[HPOLIB]
+function = python ../branin.py
+number_of_jobs = 200
+result_on_terminate = 1000
diff --git a/benchmarks/branin/tpe/space.py b/benchmarks/branin/hyperopt_august2013_mod/space.py
similarity index 100%
rename from benchmarks/branin/tpe/space.py
rename to benchmarks/branin/hyperopt_august2013_mod/space.py
diff --git a/benchmarks/branin/random_hyperopt_august2013_mod b/benchmarks/branin/random_hyperopt_august2013_mod
new file mode 120000
index 00000000..398d85cc
--- /dev/null
+++ b/benchmarks/branin/random_hyperopt_august2013_mod
@@ -0,0 +1 @@
+hyperopt_august2013_mod/
\ No newline at end of file
diff --git a/benchmarks/branin/smac/params.txt b/benchmarks/branin/smac_2_06_01-dev/params.pcs
similarity index 100%
rename from benchmarks/branin/smac/params.txt
rename to benchmarks/branin/smac_2_06_01-dev/params.pcs
diff --git a/benchmarks/branin/spearmint_april2013_mod/config.pb b/benchmarks/branin/spearmint_april2013_mod/config.pb
new file mode 100644
index 00000000..ddf59faa
--- /dev/null
+++ b/benchmarks/branin/spearmint_april2013_mod/config.pb
@@ -0,0 +1,18 @@
+language: PYTHON
+name: "HPOlib.cv"
+
+variable {
+ name: "x"
+ type: FLOAT
+ size: 1
+ min: -5
+ max: 10
+}
+
+variable {
+ name: "y"
+ type: FLOAT
+ size: 1
+ min: 0
+ max: 15
+}
diff --git a/benchmarks/camelback/__init__.py b/benchmarks/camelback/__init__.py
new file mode 100644
index 00000000..24cd9aaa
--- /dev/null
+++ b/benchmarks/camelback/__init__.py
@@ -0,0 +1,2 @@
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
\ No newline at end of file
diff --git a/benchmarks/camelback/camelback.rb b/benchmarks/camelback/camelback.rb
new file mode 100644
index 00000000..d597d1a5
--- /dev/null
+++ b/benchmarks/camelback/camelback.rb
@@ -0,0 +1,46 @@
+#! /usr/bin/ruby
+
+# ruby camelback.rb --fold 2 --folds 20 --params -x -0.0898 -y 0.7126
+# >> Result for ParamILS: SAT, 0.000095, 1, -1.031628, -1, camelback.rb
+# -2 < x < 2, -1 < y < 1
+# 2 Global minima: f(x) = -1.0316 (0.0898, -0.7126), (-0.0898, 0.7126)
+
+def camelback(x, y)
+ puts 'Params: ', x, y
+
+ tmp1 = (4 - 2.1 * x**2 + (x**4)/3) * x**2
+ tmp2 = x*y
+ tmp3 = (-4 + 4 * y**2) * y**2
+ y = tmp1 + tmp2 + tmp3
+
+ puts 'Result: ', y
+ return y
+end
+
+starttime = Time.now()
+x = -1
+y = -1
+puts ARGV
+if ARGV[5] == '-y'
+ y = ARGV[6].to_f
+elsif ARGV[5] == '-x'
+ x = ARGV[6].to_f
+else
+
+ abort("Result for ParamILS: CRASH, 1, 1, 10, -1, camelback.rb")
+end
+
+if ARGV[7] == '-y'
+ y = ARGV[8].to_f
+elsif ARGV[7] == '-x'
+ x = ARGV[8].to_f
+else
+ abort("Result for ParamILS: CRASH, 1, 1, 10, -1, camelback.rb")
+end
+
+result = camelback(x, y)
+
+# Benchmark need to take some time, because of a bug in runsolver
+sleep(1)
+duration = Time.now() - starttime
+printf "\nResult for ParamILS: %s, %f, 1, %f, %d, %s\n", "SAT", duration, result, -1, "camelback.rb"
\ No newline at end of file
diff --git a/benchmarks/camelback/config.cfg b/benchmarks/camelback/config.cfg
new file mode 100644
index 00000000..2cb8de8e
--- /dev/null
+++ b/benchmarks/camelback/config.cfg
@@ -0,0 +1,13 @@
+[SMAC]
+p = smac_2_06_01-dev/params.pcs
+
+[TPE]
+space = space.py
+
+[SPEARMINT]
+config = config.pb
+
+[HPOLIB]
+function = ruby ../camelback.rb
+number_of_jobs = 100
+result_on_terminate = 10
diff --git a/benchmarks/camelback/hyperopt_august2013_mod/space.py b/benchmarks/camelback/hyperopt_august2013_mod/space.py
new file mode 100644
index 00000000..3a42ba46
--- /dev/null
+++ b/benchmarks/camelback/hyperopt_august2013_mod/space.py
@@ -0,0 +1,4 @@
+from hyperopt import hp
+
+space = {'x': hp.uniform('x', -2, 2),
+ 'y': hp.uniform('y', -1, 1)}
\ No newline at end of file
diff --git a/benchmarks/camelback/random_hyperopt_august2013_mod b/benchmarks/camelback/random_hyperopt_august2013_mod
new file mode 120000
index 00000000..398d85cc
--- /dev/null
+++ b/benchmarks/camelback/random_hyperopt_august2013_mod
@@ -0,0 +1 @@
+hyperopt_august2013_mod/
\ No newline at end of file
diff --git a/benchmarks/camelback/smac_2_06_01-dev/params.pcs b/benchmarks/camelback/smac_2_06_01-dev/params.pcs
new file mode 100644
index 00000000..d42b73a6
--- /dev/null
+++ b/benchmarks/camelback/smac_2_06_01-dev/params.pcs
@@ -0,0 +1,2 @@
+x [-2, 2] [0]
+y [-1, 1] [0]
diff --git a/benchmarks/camelback/spearmint_april2013_mod/config.pb b/benchmarks/camelback/spearmint_april2013_mod/config.pb
new file mode 100644
index 00000000..13648b4a
--- /dev/null
+++ b/benchmarks/camelback/spearmint_april2013_mod/config.pb
@@ -0,0 +1,18 @@
+language: PYTHON
+name: "HPOlib.cv"
+
+variable {
+ name: "x"
+ type: FLOAT
+ size: 1
+ min: -2
+ max: 2
+}
+
+variable {
+ name: "y"
+ type: FLOAT
+ size: 1
+ min: -1
+ max: 1
+}
diff --git a/benchmarks/har6/__init__.py b/benchmarks/har6/__init__.py
new file mode 100644
index 00000000..24cd9aaa
--- /dev/null
+++ b/benchmarks/har6/__init__.py
@@ -0,0 +1,2 @@
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
\ No newline at end of file
diff --git a/benchmarks/har6/config.cfg b/benchmarks/har6/config.cfg
index 8354d2d1..8ff970e2 100644
--- a/benchmarks/har6/config.cfg
+++ b/benchmarks/har6/config.cfg
@@ -1,9 +1,9 @@
[SMAC]
-p = smac/params.txt
+p = smac_2_06_01-dev/params.txt
-[DEFAULT]
-function = har6.py
-numberOfJobs = 200
+[HPOLIB]
+function = python ../har6.py
+number_of_jobs = 200
result_on_terminate = 1
# Overwrite
diff --git a/benchmarks/har6/har6.py b/benchmarks/har6/har6.py
index ac0aa0c6..f7054f6e 100644
--- a/benchmarks/har6/har6.py
+++ b/benchmarks/har6/har6.py
@@ -16,75 +16,28 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-from numpy import NaN, array, ndarray, exp
-import benchmark_util
+import sys
+import time
+
+import HPOlib.benchmark_util as benchmark_util
+import HPOlib.benchmark_functions as benchmark_functions
__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
__contact__ = "automl.org"
-
-
-def har6(params, **kwargs):
-# 6d Hartmann function
-# constraints:
-# 0 <= xi <= 1, i = 1..6
-# global optimum at (0.20169, 0.150011, 0.476874, 0.275332, 0.311652, 0.6573),
-# where har6 = -3.32236
- if "x" not in params or "y" not in params or "z" not in params \
- or "a" not in params or "b" not in params or "c" not in params:
- sys.stderr.write("No params found ['x', 'y']\n")
- return NaN
- x = float(params["x"])
- y = float(params["y"])
- z = float(params["z"])
- a = float(params["a"])
- b = float(params["b"])
- c = float(params["c"])
- if type(x) == ndarray:
- x = x[0]
- y = y[0]
- z = z[0]
- a = a[0]
- b = b[0]
- c = c[0]
- value = array([x, y, z, a, b, c])
-
- if 0 > x or x > 1 or 0 > y or y > 1 or 0 > z or z > 1:
- raise ValueError("x=%s, y=%s or z=%s not in between 0 and 1" %
- (x, y, z))
- if 0 > a or a > 1 or 0 > b or b > 1 or 0 > c or c > 1:
- raise ValueError("a=%s, b=%s or c=%s not in between 0 and 1" %
- (a, b, c))
-
- a = array([[10.0, 3.0, 17.0, 3.5, 1.7, 8.0],
- [ 0.05, 10.0, 17.0, 0.1, 8.0, 14.0],
- [ 3.0, 3.5, 1.7, 10.0, 17.0, 8.0],
- [17.0, 8.0, 0.05, 10.0, 0.1, 14.0]])
- c = array([1.0, 1.2, 3.0, 3.2])
- p = array([[0.1312, 0.1696, 0.5569, 0.0124, 0.8283, 0.5886],
- [0.2329, 0.4135, 0.8307, 0.3736, 0.1004, 0.9991],
- [0.2348, 0.1451, 0.3522, 0.2883, 0.3047, 0.6650],
- [0.4047, 0.8828, 0.8732, 0.5743, 0.1091, 0.0381]])
- s = 0
- for i in [0,1,2,3]:
- sm = a[i,0]*(value[0]-p[i,0])**2
- sm += a[i,1]*(value[1]-p[i,1])**2
- sm += a[i,2]*(value[2]-p[i,2])**2
- sm += a[i,3]*(value[3]-p[i,3])**2
- sm += a[i,4]*(value[4]-p[i,4])**2
- sm += a[i,5]*(value[5]-p[i,5])**2
- s += c[i]*exp(-sm)
- result = -s
- return result
def main(params, **kwargs):
print 'Params: ', params,
- y = har6(params, **kwargs)
+ y = benchmark_functions.save_har6(params, **kwargs)
print 'Result: ', y
return y
if __name__ == "__main__":
+ print sys.argv
+ starttime = time.time()
args, params = benchmark_util.parse_cli()
result = main(params, **args)
- print "Result", result
\ No newline at end of file
+ duration = time.time() - starttime
+ print "Result for ParamILS: %s, %f, 1, %f, %d, %s" % \
+ ("SAT", abs(duration), result, -1, str(__file__))
diff --git a/benchmarks/har6/tpe/space.py b/benchmarks/har6/hyperopt_august2013_mod/space.py
similarity index 100%
rename from benchmarks/har6/tpe/space.py
rename to benchmarks/har6/hyperopt_august2013_mod/space.py
diff --git a/benchmarks/har6/random_hyperopt_august2013_mod b/benchmarks/har6/random_hyperopt_august2013_mod
new file mode 120000
index 00000000..398d85cc
--- /dev/null
+++ b/benchmarks/har6/random_hyperopt_august2013_mod
@@ -0,0 +1 @@
+hyperopt_august2013_mod/
\ No newline at end of file
diff --git a/benchmarks/har6/smac/params.txt b/benchmarks/har6/smac_2_06_01-dev/params.txt
similarity index 100%
rename from benchmarks/har6/smac/params.txt
rename to benchmarks/har6/smac_2_06_01-dev/params.txt
diff --git a/benchmarks/har6/spearmint/config.pb b/benchmarks/har6/spearmint_april2013_mod/config.pb
similarity index 94%
rename from benchmarks/har6/spearmint/config.pb
rename to benchmarks/har6/spearmint_april2013_mod/config.pb
index d051740a..4d4ed4e0 100644
--- a/benchmarks/har6/spearmint/config.pb
+++ b/benchmarks/har6/spearmint_april2013_mod/config.pb
@@ -1,5 +1,5 @@
language: PYTHON
-name: "cv"
+name: "HPOlib.cv"
variable {
name: "x"
diff --git a/benchmarks/lda_on_grid/config.cfg b/benchmarks/lda_on_grid/config.cfg
new file mode 100644
index 00000000..6c245bdb
--- /dev/null
+++ b/benchmarks/lda_on_grid/config.cfg
@@ -0,0 +1,13 @@
+[SMAC]
+p = smac_2_06_01-dev/params.pcs
+
+[TPE]
+space = space.py
+
+[SPEARMINT]
+config = config.pb
+
+[HPOLIB]
+function = python ../lda.py
+number_of_jobs = 50
+result_on_terminate = 1000000000
diff --git a/benchmarks/lda_on_grid/hyperopt_august2013_mod/space.py b/benchmarks/lda_on_grid/hyperopt_august2013_mod/space.py
new file mode 100644
index 00000000..3b8ead30
--- /dev/null
+++ b/benchmarks/lda_on_grid/hyperopt_august2013_mod/space.py
@@ -0,0 +1,5 @@
+from hyperopt import hp
+
+space = {'Kappa': hp.quniform('Kappa', 0, 5, 1),
+ 'Tau': hp.quniform('Tau', 0, 5, 1),
+ 'S': hp.quniform('S', 0, 7, 1)}
diff --git a/benchmarks/lda_on_grid/lda.py b/benchmarks/lda_on_grid/lda.py
new file mode 100644
index 00000000..c21a1125
--- /dev/null
+++ b/benchmarks/lda_on_grid/lda.py
@@ -0,0 +1,40 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+
+import HPOlib.benchmark_util as benchmark_util
+import HPOlib.benchmark_functions as benchmark_functions
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+__credits__ = ["Jasper Snoek", "Ryan P. Adams", "Hugo Larochelle"]
+
+
+def main(params, ret_time=False, **kwargs):
+ print 'Params: ', params
+ y = benchmark_functions.save_lda_on_grid(params, ret_time=ret_time, **kwargs)
+ print 'Result: ', y
+ return y
+
+
+if __name__ == "__main__":
+ args, params = benchmark_util.parse_cli()
+ result = main(params, ret_time=False, **args)
+ duration = main(params, ret_time=True)
+ print "Result for ParamILS: %s, %f, 1, %f, %d, %s" % \
+ ("SAT", abs(duration), result, -1, str(__file__))
\ No newline at end of file
diff --git a/benchmarks/lda_on_grid/random_hyperopt_august2013_mod b/benchmarks/lda_on_grid/random_hyperopt_august2013_mod
new file mode 120000
index 00000000..c4f9ce32
--- /dev/null
+++ b/benchmarks/lda_on_grid/random_hyperopt_august2013_mod
@@ -0,0 +1 @@
+hyperopt_august2013_mod
\ No newline at end of file
diff --git a/benchmarks/lda_on_grid/smac_2_06_01-dev/params.pcs b/benchmarks/lda_on_grid/smac_2_06_01-dev/params.pcs
new file mode 100644
index 00000000..c3551e94
--- /dev/null
+++ b/benchmarks/lda_on_grid/smac_2_06_01-dev/params.pcs
@@ -0,0 +1,4 @@
+Kappa [0, 5] [0]i
+Tau [0, 5] [0]i
+S [0, 7] [0]i
+
diff --git a/benchmarks/lda_on_grid/spearmint_april2013_mod/config.pb b/benchmarks/lda_on_grid/spearmint_april2013_mod/config.pb
new file mode 100644
index 00000000..0dff3cdb
--- /dev/null
+++ b/benchmarks/lda_on_grid/spearmint_april2013_mod/config.pb
@@ -0,0 +1,27 @@
+language: PYTHON
+name: "HPOlib.cv"
+
+variable {
+ name: "Kappa"
+ type: INT
+ size: 1
+ min: 0
+ max: 5
+}
+
+variable {
+ name: "Tau"
+ type: INT
+ size: 1
+ min: 0
+ max: 5
+}
+
+variable {
+ name: "S"
+ type: INT
+ size: 1
+ min: 0
+ max: 7
+}
+
diff --git a/benchmarks/svm_on_grid/config.cfg b/benchmarks/svm_on_grid/config.cfg
new file mode 100644
index 00000000..1eb3085f
--- /dev/null
+++ b/benchmarks/svm_on_grid/config.cfg
@@ -0,0 +1,13 @@
+[SMAC]
+p = smac_2_06_01-dev/params.pcs
+
+[TPE]
+space = space.py
+
+[SPEARMINT]
+config = config.pb
+
+[HPOLIB]
+function = python ../svm.py
+number_of_jobs = 100
+result_on_terminate = 1
diff --git a/benchmarks/svm_on_grid/hyperopt_august2013_mod/random_hyperopt_august2013_mod b/benchmarks/svm_on_grid/hyperopt_august2013_mod/random_hyperopt_august2013_mod
new file mode 120000
index 00000000..6b6e627b
--- /dev/null
+++ b/benchmarks/svm_on_grid/hyperopt_august2013_mod/random_hyperopt_august2013_mod
@@ -0,0 +1 @@
+random_hyperopt_august2013_mod
\ No newline at end of file
diff --git a/benchmarks/svm_on_grid/hyperopt_august2013_mod/space.py b/benchmarks/svm_on_grid/hyperopt_august2013_mod/space.py
new file mode 100644
index 00000000..f2a2a583
--- /dev/null
+++ b/benchmarks/svm_on_grid/hyperopt_august2013_mod/space.py
@@ -0,0 +1,5 @@
+from hyperopt import hp
+
+space = {'C': hp.quniform('C', 0, 24, 1),
+ 'alpha': hp.quniform('alpha', 0, 13, 1),
+ 'epsilon': hp.quniform('epsilon', 0, 3, 1)}
diff --git a/benchmarks/svm_on_grid/random_hyperopt_august2013_mod b/benchmarks/svm_on_grid/random_hyperopt_august2013_mod
new file mode 120000
index 00000000..c4f9ce32
--- /dev/null
+++ b/benchmarks/svm_on_grid/random_hyperopt_august2013_mod
@@ -0,0 +1 @@
+hyperopt_august2013_mod
\ No newline at end of file
diff --git a/benchmarks/svm_on_grid/smac_2_06_01-dev/params.pcs b/benchmarks/svm_on_grid/smac_2_06_01-dev/params.pcs
new file mode 100644
index 00000000..bfd0367a
--- /dev/null
+++ b/benchmarks/svm_on_grid/smac_2_06_01-dev/params.pcs
@@ -0,0 +1,4 @@
+C [0, 24] [0]i
+alpha [0, 13] [0]i
+epsilon [0, 3] [0]i
+
diff --git a/benchmarks/svm_on_grid/spearmint_april2013_mod/config.pb b/benchmarks/svm_on_grid/spearmint_april2013_mod/config.pb
new file mode 100644
index 00000000..2df3dff7
--- /dev/null
+++ b/benchmarks/svm_on_grid/spearmint_april2013_mod/config.pb
@@ -0,0 +1,27 @@
+language: PYTHON
+name: "HPOlib.cv"
+
+variable {
+ name: "C"
+ type: INT
+ size: 1
+ min: 0
+ max: 24
+}
+
+variable {
+ name: "alpha"
+ type: INT
+ size: 1
+ min: 0
+ max: 13
+}
+
+variable {
+ name: "epsilon"
+ type: INT
+ size: 1
+ min: 0
+ max: 3
+}
+
diff --git a/benchmarks/svm_on_grid/svm.py b/benchmarks/svm_on_grid/svm.py
new file mode 100644
index 00000000..61e3100b
--- /dev/null
+++ b/benchmarks/svm_on_grid/svm.py
@@ -0,0 +1,41 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import time
+
+import HPOlib.benchmark_util as benchmark_util
+import HPOlib.benchmark_functions as benchmark_functions
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+__credits__ = ["Jasper Snoek", "Ryan P. Adams", "Hugo Larochelle"]
+
+
+def main(params, ret_time=False, **kwargs):
+ print 'Params: ', params, '\n'
+ y = benchmark_functions.save_svm_on_grid(params, opt_time=ret_time, **kwargs)
+ print 'Result: ', y
+ return y
+
+
+if __name__ == "__main__":
+ args, cli_params = benchmark_util.parse_cli()
+ result = main(cli_params, ret_time=False, **args)
+ duration = main(cli_params, ret_time=True, **args)
+ print "Result for ParamILS: %s, %f, 1, %f, %d, %s" % \
+ ("SAT", abs(duration), result, -1, str(__file__))
\ No newline at end of file
diff --git a/check_before_start.py b/check_before_start.py
deleted file mode 100644
index d965dd9d..00000000
--- a/check_before_start.py
+++ /dev/null
@@ -1,146 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-#!/usr/bin/env python
-
-import os
-import subprocess
-import sys
-import imp
-from config_parser.parse import parse_config
-
-"""This script checks whether all dependencies are installed"""
-
-
-def _check_runsolver():
- # check whether runsolver is in path
- process = subprocess.Popen("which runsolver", stdout=subprocess.PIPE,
- stderr=subprocess.PIPE, shell=True,
- executable="/bin/bash")
- stdoutdata, stderrdata = process.communicate()
- if stdoutdata is not None and "runsolver" in stdoutdata:
- pass
- else:
- raise Exception("Runsolver cannot not be found. Are you sure that it's installed?")
-
-
-def _check_modules():
- try:
- import numpy
- if numpy.__version__ < "1.6.0":
- print "WARNING: You are using a numpy %s < 1.6.0. This might not work"\
- % numpy.__version__
- except:
- raise ImportError("Numpy cannot be imported. Are you sure that it's installed?")
-
- try:
- import scipy
- if scipy.__version__ < "0.12.0":
- print "WARNING: You are using a scipy %s < 0.12.0. This might not work"\
- % scipy.__version__
- except:
- raise ImportError("Scipy cannot be imported. Are you sure that it's installed?")
- import bson
- import networkx
- import google.protobuf
-
- try:
- import theano
- except ImportError:
- print "Theano not found"
-
- if 'cuda' not in os.environ['PATH']:
- print "CUDA not in $PATH"
- # if 'cuda' not in os.environ['LD_LIBRARY_PATH']:
- # print "CUDA not in $LD_LIBRARY_PATH"
-
-
-def _check_config(experiment_dir):
- # check whether config file exists
- config_file = os.path.join(experiment_dir, "config.cfg")
- if not os.path.exists(config_file):
- raise Exception("There is no config.cfg in %s" % experiment_dir)
-
-
-def _check_function(experiment_dir, optimizer_dir):
- # Check whether function exists
- config_file = os.path.join(experiment_dir, "config.cfg")
- config = parse_config(config_file, allow_no_value=True)
-
- fn = None
- if os.path.isabs(config.get("DEFAULT", "function")):
- fn_path = config.get("DEFAULT", "function")
- fn_name, ext = os.path.splitext(os.path.basename(fn_path))
- try:
- fn = imp.load_source(fn_name, fn_path)
- except (ImportError, IOError):
- print "Could not find algorithm in %s" % fn_path
- import traceback
- print traceback.format_exc()
- else:
- fn = config.get("DEFAULT", "function").replace("../..", "..", 1)
- fn_path = os.path.join(optimizer_dir, fn)
- fn_path_parent = os.path.join(optimizer_dir, "..", fn)
- fn_name, ext = os.path.splitext(os.path.basename(fn_path))
- try:
- fn = imp.load_source(fn_name, fn_path)
- except (ImportError, IOError) as e:
- print e
- try:
- fn = imp.load_source(fn_name, fn_path_parent)
- except IOError as e:
- print(("Could not find\n%s\n\tin\n%s\n\tor its parent directory " +
- "\n%s")
- % (fn_name, fn_path, fn_path_parent))
- import traceback
- print traceback.format_exc()
- sys.exit(1)
- except ImportError as e:
- import traceback
- print traceback.format_exc()
- sys.exit(1)
- del fn
-
-
-def _check_first(experiment_dir):
- """ Do some checks before optimizer is loaded """
- main()
- print "\tconfig.cfg..",
- _check_config(experiment_dir)
- print "..passed"
-
-
-def _check_second(experiment_dir, optimizer_dir):
- """ Do remaining tests """
- print "\talgorithm..",
- _check_function(experiment_dir, optimizer_dir)
- print "..passed"
-
-
-def main():
- print "Checking dependencies:"
- print "\tRunsolver..",
- _check_runsolver()
- print "..passed"
- print "\tpython_modules..",
- _check_modules()
- print "..passed"
-
-
-if __name__ == "__main__":
- main()
\ No newline at end of file
diff --git a/config_parser/generalDefault.cfg b/config_parser/generalDefault.cfg
deleted file mode 100644
index 271f34dc..00000000
--- a/config_parser/generalDefault.cfg
+++ /dev/null
@@ -1,28 +0,0 @@
-[DEFAULT]
-#Will be used for wrapping.py and SMAC
-algorithm = cv.py
-run_instance = runsolver_wrapper.py
-numberCV = 1
-max_crash_per_cv = 3
-
-numberOfConcurrentJobs = 1
-
-#The mem-limit must be expressed in mega-bytes
-#The cpu-limit must be expressed in seconds (CPU time)
-#The time-limit must be expressed in seconds (wall clock time)
-runsolver_time_limit = 3600
-total_time_limit = 3600
-memory_limit = 2000
-cpu_limit = 14400
-
-# E.g. Information for Theano and cuda
-leading_runsolver_info =
-leading_algo_info =
-
-training_data =
-training_targets =
-test_data =
-test_targets =
-#only gfile, numpy and pickle supported
-training_data_format = numpy
-use_percentage = 100
diff --git a/config_parser/parse.py b/config_parser/parse.py
deleted file mode 100644
index ae82263b..00000000
--- a/config_parser/parse.py
+++ /dev/null
@@ -1,103 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-#!/usr/bin/env python
-
-import os
-import imp
-import sys
-import ConfigParser
-
-
-def parse_config(config_fn, allow_no_value=True, optimizer_module=""):
- # Reads config_fn
- # Overwrites with default values from generalDefault.cfg
- # Loads optimizer specific parser, called 'optimizer_module'_parser.py, which can read its own default config
- if not os.path.isfile(config_fn):
- raise Exception('%s is not a valid file\n' % config_fn)
-
- config = ConfigParser.SafeConfigParser(allow_no_value=allow_no_value)
- config.read(config_fn)
-
- # Load general default configs
- config_fn_default = \
- os.path.join(os.path.dirname(os.path.realpath(__file__)), "generalDefault.cfg")
- config_default = ConfigParser.SafeConfigParser(allow_no_value=True)
-
- if not os.path.isfile(config_fn_default):
- raise Exception('%s does not exist\n' % config_fn_default)
- config_default.read(config_fn_default)
-
- # --------------------------------------------------------------------------
- # DEFAULT
- # --------------------------------------------------------------------------
- # Set defaults for DEFAULT
- for option in ('algorithm', 'run_instance', 'numberCV',
- 'leading_algo_info', 'numberOfConcurrentJobs',
- 'runsolver_time_limit', 'total_time_limit', 'memory_limit',
- 'cpu_limit', 'leading_runsolver_info', 'training_data',
- 'training_targets', 'test_data', 'test_targets',
- 'training_data_format', 'use_percentage', 'max_crash_per_cv'):
- if not config.has_option('DEFAULT', option):
- config.set('DEFAULT', option,
- config_default.get('DEFAULT', option))
-
- if not config.has_option('DEFAULT', 'numberOfJobs') or \
- config.get('DEFAULT', 'numberOfJobs') == '':
- raise Exception('numberOfJobs not specified in .cfg')
- if not config.has_option('DEFAULT', 'result_on_terminate') or \
- config.get('DEFAULT', 'result_on_terminate') == '':
- raise Exception('No result_on_terminate specified in .cfg')
- if not config.has_option('DEFAULT', 'function') or \
- config.get('DEFAULT', 'function') == '':
- raise Exception('No function specified in .cfg')
-
- # Load optimizer parsing module
- if optimizer_module == "":
- return config
- optimizer_module_name = optimizer_module + "_parser"
- optimizer_module_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), optimizer_module_name + ".py")
- try:
- optimizer_module_loaded = imp.load_source(optimizer_module_name, optimizer_module_path)
- except Exception, e:
- print('Could not find\n%s\n\tin\n%s\n\t relative to\n%s'
- % (optimizer_module_name, optimizer_module_path, os.getcwd()))
- import traceback
- print(traceback.format_exc())
- sys.exit(1)
- # Add optimizer specific defaults
- config = optimizer_module_loaded.add_default(config)
-
- return config
-
-
-def main():
- config_fn = sys.argv[1]
- print 'Read config from %s..' % config_fn
- print '\twith spearmint..',
- parse_config(config_fn, optimizer_module="spearmint")
- print '..finished\n\twith smac..',
- parse_config(config_fn, optimizer_module="smac")
- print '..finished\n\twith tpe..',
- parse_config(config_fn, optimizer_module="tpe")
- print '..finished\n..finished'
-
-if __name__ == "__main__":
- main()
-
-
diff --git a/config_parser/smacDefault.cfg b/config_parser/smacDefault.cfg
deleted file mode 100644
index e9830248..00000000
--- a/config_parser/smacDefault.cfg
+++ /dev/null
@@ -1,27 +0,0 @@
-[SMAC]
-# cutoffTime = runsolver_time_limit + 100 sec
-
-# Set otherwise
-# algoExec = %(run_instance)s
-# numConcurrentAlgoExecs = %(numberOfConcurrentJobs)s
-# experimentDir = %(exp_directory)s/
-# outputDirectory = %(exp_directory)s/smac/output/
-# totalNumRunsLimit = %(numberOfJobs)s * %(cv)s
-
-numRun = 0
-intraInstanceObj = MEAN
-runObj = QUALITY
-instanceFile = train.txt
-testInstanceFile = test.txt
-#execDir = ./
-p = smac/params.pcs
-rf_full_tree_bootstrap = False
-rf_split_min = 10
-
-adaptiveCapping = false
-maxIncumbentRuns = 2000
-numIterations = 2147483647
-runtimeLimit = 2147483647
-
-deterministic = True
-retryTargetAlgorithmRunCount = 0
\ No newline at end of file
diff --git a/config_parser/smac_parser.py b/config_parser/smac_parser.py
deleted file mode 100644
index 8f5e4054..00000000
--- a/config_parser/smac_parser.py
+++ /dev/null
@@ -1,67 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-import os
-
-import ConfigParser
-
-
-def add_default(config):
- # This module reads smacDefault.cfg and adds this defaults to a given config
-
- assert isinstance(config, ConfigParser.RawConfigParser), \
- 'config is not a valid instance'
-
- config_fn = os.path.join(os.path.dirname(os.path.realpath(__file__)),
- "smacDefault.cfg")
- if not os.path.isfile(config_fn):
- raise Exception('%s is not a valid file\n' % config_fn)
-
- smac_config = ConfigParser.SafeConfigParser(allow_no_value=True)
- smac_config.read(config_fn)
- # --------------------------------------------------------------------------
- # SMAC
- # --------------------------------------------------------------------------
- # Set defaults for SMAC
- if not config.has_section('SMAC'):
- config.add_section('SMAC')
- # optional arguments (exec_dir taken out as is does not seem to be used)
- for option in ('numRun', 'instanceFile', 'intraInstanceObj', 'runObj',
- 'testInstanceFile', 'p', 'rf_full_tree_bootstrap',
- 'rf_split_min', 'adaptiveCapping', 'maxIncumbentRuns',
- 'numIterations', 'runtimeLimit', 'deterministic',
- 'retryTargetAlgorithmRunCount'):
- if not config.has_option('SMAC', option):
- config.set('SMAC', option,
- smac_config.get('SMAC', option))
-
- # special cases
- if not config.has_option('SMAC', 'cutoffTime'):
- config.set('SMAC', 'cutoffTime',
- str(config.getint('DEFAULT', 'runsolver_time_limit') + 100))
- if not config.has_option('SMAC', 'algoExec'):
- config.set('SMAC', 'algoExec',
- config.get('DEFAULT', 'run_instance'))
- if not config.has_option('SMAC', 'totalNumRunsLimit'):
- config.set('SMAC', 'totalNumRunsLimit',
- str(config.getint('DEFAULT', 'numberOfJobs') *
- config.getint('DEFAULT', 'numberCV')))
- if not config.has_option('SMAC', 'numConcurrentAlgoExecs'):
- config.set('SMAC', 'numConcurrentAlgoExecs',
- config.get('DEFAULT', 'numberOfConcurrentJobs'))
- return config
\ No newline at end of file
diff --git a/config_parser/spearmint_parser.py b/config_parser/spearmint_parser.py
deleted file mode 100644
index 44e2fca6..00000000
--- a/config_parser/spearmint_parser.py
+++ /dev/null
@@ -1,58 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-import os
-
-import ConfigParser
-
-
-def add_default(config):
- # This module reads spearmintDefault.cfg and adds this defaults to a given config
- assert isinstance(config, ConfigParser.RawConfigParser), "config is not a valid instance"
-
- config_fn = os.path.join(os.path.dirname(os.path.realpath(__file__)), "spearmintDefault.cfg")
- if not os.path.isfile(config_fn):
- raise Exception('%s is not a valid file\n' % config_fn)
-
- spearmint_config = ConfigParser.SafeConfigParser(allow_no_value=True)
- spearmint_config.read(config_fn)
- # --------------------------------------------------------------------------
- # SPEARMINT
- # --------------------------------------------------------------------------
- # Set default for SPEARMINT
- if not config.has_section('SPEARMINT'):
- config.add_section('SPEARMINT')
- # optional arguments
- for option in ('script', 'method', 'grid_size', 'config', 'grid_seed',
- 'spearmint_polling_time', 'max_concurrent', 'method_args'):
- if not config.has_option('SPEARMINT', option):
- config.set('SPEARMINT', option,
- spearmint_config.get('SPEARMINT', option))
-
- # special cases
- if not config.has_option('SPEARMINT', 'method'):
- config.set('SPEARMINT', 'method',
- spearmint_config.get('SPEARMINT', 'method'))
- config.set('SPEARMINT', 'method-args',
- spearmint_config.get('SPEARMINT', 'method-args'))
-
- # GENERAL
- if not config.has_option('SPEARMINT', 'max_finished_jobs'):
- config.set('SPEARMINT', 'max_finished_jobs',
- config.get('DEFAULT', 'numberOfJobs'))
- return config
\ No newline at end of file
diff --git a/config_parser/tpeDefault.cfg b/config_parser/tpeDefault.cfg
deleted file mode 100644
index 57565812..00000000
--- a/config_parser/tpeDefault.cfg
+++ /dev/null
@@ -1,4 +0,0 @@
-[TPE]
-space = space.py
-#exp_dir = %(exp_directory)s/tpe/
-#numberEvals = %(numberOfJobs)s
\ No newline at end of file
diff --git a/config_parser/tpe_parser.py b/config_parser/tpe_parser.py
deleted file mode 100644
index 2db6dc25..00000000
--- a/config_parser/tpe_parser.py
+++ /dev/null
@@ -1,51 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-import os
-
-import ConfigParser
-
-
-def add_default(config):
- # This module reads tpeDefault.cfg and adds this defaults to a given config
- assert isinstance(config, ConfigParser.RawConfigParser), \
- "config is not a valid instance"
-
- config_fn = os.path.join(os.path.dirname(os.path.realpath(__file__)),
- "tpeDefault.cfg")
- if not os.path.isfile(config_fn):
- raise Exception('%s is not a valid file\n' % config_fn)
-
- tpe_config = ConfigParser.SafeConfigParser(allow_no_value=True)
- tpe_config.read(config_fn)
- # --------------------------------------------------------------------------
- # TPE
- # --------------------------------------------------------------------------
- # Set default for TPE
- if not config.has_section('TPE'):
- config.add_section('TPE')
-
- # optional cases
- if not config.has_option('TPE', 'space'):
- config.set('TPE', 'space',
- tpe_config.get('TPE', 'space'))
-
- if not config.has_option('TPE', 'numberEvals'):
- config.set('TPE', 'numberEvals',
- config.get('DEFAULT', 'numberOfJobs'))
- return config
\ No newline at end of file
diff --git a/smac.py b/optimizers/smac/smac_2_06_01-dev.py
similarity index 72%
rename from smac.py
rename to optimizers/smac/smac_2_06_01-dev.py
index fdac42d7..2b038d55 100644
--- a/smac.py
+++ b/optimizers/smac/smac_2_06_01-dev.py
@@ -16,9 +16,8 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-#!/usr/bin/env python
-
import glob
+import logging
import os
import re
import subprocess
@@ -26,16 +25,33 @@
import numpy as np
-import wrapping_util
+import HPOlib.wrapping_util as wrapping_util
+
+
+logger = logging.getLogger("HPOlib.smac_2_06_01-dev")
+
__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
__contact__ = "automl.org"
-path_to_optimizer = "optimizers/smac_2_06_01-dev"
-version_info = ("# %76s #\n# %76s #\n# %76s #\n" %
- ("Automatic Configurator Library ==> v2.06.01-development-643 (a1f71813a262)",
+version_info = ["Automatic Configurator Library ==> v2.06.01-development-643 (a1f71813a262)",
"Random Forest Library ==> v1.05.01-development-95 (4a8077e95b21)",
- "SMAC ==> v2.06.01-development-620 (9380d2c6bab9)"))
+ "SMAC ==> v2.06.01-development-620 (9380d2c6bab9)"]
+
+#optimizer_str = "smac_2_06_01-dev"
+
+
+def check_dependencies():
+ process = subprocess.Popen("which java", stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, shell=True, executable="/bin/bash")
+ stdoutdata, stderrdata = process.communicate()
+
+ if stdoutdata is not None and "java" in stdoutdata:
+ pass
+ else:
+ raise Exception("Java cannot not be found. "
+ "Are you sure that it's installed?\n"
+ "Your $PATH is: " + os.environ['PATH'])
def _get_state_run(optimizer_dir):
@@ -43,13 +59,13 @@ def _get_state_run(optimizer_dir):
if len(rungroups) == 1:
rungroup = rungroups[0]
else:
- print "Found multiple rungroups, take the newest one."
+ logger.warning("Found multiple rungroups, take the newest one.")
creation_times = []
for i, filename in enumerate(rungroups):
creation_times.append(float(os.path.getctime(filename)))
newest = np.argmax(creation_times)
rungroup = rungroups[newest]
- print creation_times, newest, rungroup
+ logger.info(creation_times, newest, rungroup)
state_runs = glob.glob(rungroup + "/state-run*")
if len(state_runs) != 1:
@@ -60,39 +76,45 @@ def _get_state_run(optimizer_dir):
def build_smac_call(config, options, optimizer_dir):
- call = os.path.dirname(os.path.realpath(__file__)) + "/" + path_to_optimizer + "/smac"
+ import HPOlib
+ algo_exec_dir = os.path.dirname(HPOlib.__file__)
+
+ call = config.get('SMAC', 'path_to_optimizer') + "/smac"
call = " ".join([call, '--numRun', str(options.seed),
'--scenario-file', os.path.join(optimizer_dir, 'scenario.txt'),
- '--cutoffTime', config.get('SMAC', 'cutoffTime'),
+ '--cutoffTime', config.get('SMAC', 'cutoff_time'),
# The instance file does interfere with state restoration, it will only
# be loaded if no state is restored (look further down in the code
# '--instanceFile', config.get('SMAC', 'instanceFile'),
- '--intraInstanceObj', config.get('SMAC', 'intraInstanceObj'),
- '--runObj', config.get('SMAC', 'runObj'),
+ '--intraInstanceObj', config.get('SMAC', 'intra_instance_obj'),
+ '--runObj', config.get('SMAC', 'run_obj'),
# '--testInstanceFile', config.get('SMAC', 'testInstanceFile'),
- '--algoExec', '"python', os.path.dirname(os.path.realpath(__file__)) + "/" +
- config.get('SMAC', 'algoExec') + '"',
+ '--algoExec', '"python', os.path.join(algo_exec_dir,
+ config.get('SMAC', 'algo_exec')) + '"',
'--execDir', optimizer_dir,
'-p', config.get('SMAC', 'p'),
# The experiment dir MUST not be specified when restarting, it is set
# further down in the code
# '--experimentDir', optimizer_dir,
- '--numIterations', config.get('SMAC', 'numIterations'),
- '--totalNumRunsLimit', config.get('SMAC', 'totalNumRunsLimit'),
+ '--numIterations', config.get('SMAC', 'num_iterations'),
+ '--totalNumRunsLimit', config.get('SMAC', 'total_num_runs_limit'),
'--outputDirectory', optimizer_dir,
- '--numConcurrentAlgoExecs', config.get('SMAC', 'numConcurrentAlgoExecs'),
+ '--numConcurrentAlgoExecs', config.get('SMAC', 'num_concurrent_algo_execs'),
# '--runGroupName', config.get('SMAC', 'runGroupName'),
- '--maxIncumbentRuns', config.get('SMAC', 'maxIncumbentRuns'),
+ '--maxIncumbentRuns', config.get('SMAC', 'max_incumbent_runs'),
'--retryTargetAlgorithmRunCount',
- config.get('SMAC', 'retryTargetAlgorithmRunCount'),
+ config.get('SMAC', 'retry_target_algorithm_run_count'),
'--save-runs-every-iteration true',
- '--rf-split-min', config.get('SMAC', 'rf_split_min')])
+ '--intensification-percentage',
+ config.get('SMAC', 'intensification_percentage'),
+ '--rf-split-min', config.get('SMAC', 'rf_split_min'),
+ '--validation', config.get('SMAC', 'validation')])
if config.getboolean('SMAC', 'deterministic'):
call = " ".join([call, '--deterministic true'])
- if config.getboolean('SMAC', 'adaptiveCapping') and \
- config.get('SMAC', 'runObj') == "RUNTIME":
+ if config.getboolean('SMAC', 'adaptive_capping') and \
+ config.get('SMAC', 'run_obj') == "RUNTIME":
call = " ".join([call, '--adaptiveCapping true'])
if config.getboolean('SMAC', 'rf_full_tree_bootstrap'):
@@ -121,17 +143,17 @@ def restore(config, optimizer_dir, **kwargs):
# Run SMAC in a manner that it restores the files but then exits
fh = open(optimizer_dir + "smac_restart.out", "w")
smac_cmd = re.sub('python ' + os.path.dirname(os.path.realpath(__file__)) +
- "/" + config.get('SMAC', 'algoExec'), 'pwd',
+ "/" + config.get('SMAC', 'algo_exec'), 'pwd',
kwargs['cmd'])
smac_cmd = re.sub('--outputDirectory ' + optimizer_dir, '--outputDirectory '
+ optimizer_dir + "restart_rungroups", smac_cmd)
- print smac_cmd
+ logger.info(smac_cmd)
process = subprocess.Popen(smac_cmd, stdout=fh, stderr=fh, shell=True,
executable="/bin/bash")
- print "-----------------------RUNNING----------------------------------"
+ logger.info("----------------------RUNNING--------------------------------")
ret = process.wait()
fh.close()
- print "Finished with return code: " + str(ret)
+ logger.info("Finished with return code: " + str(ret))
# os.remove("smac_restart.out")
# read smac.out and look how many states are restored
@@ -141,7 +163,6 @@ def restore(config, optimizer_dir, **kwargs):
for line in fh.readlines():
match = prog.search(line)
if match:
- # print "###########Match", match.group(0), match.group(2)
restored_runs = int(match.group(2))
# Find out all rungroups and state-runs
@@ -151,18 +172,13 @@ def restore(config, optimizer_dir, **kwargs):
state_run_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
os.getcwd(), state_run)
state_runs = glob.glob(state_run_path + "/runs_and_results-it*.csv")
- # print state_run_path + "/runs_and_results-it*.csv"
- # print state_runs
state_run_iterations = []
for state_run in state_runs:
match = re.search(r"(runs_and_results-it)([0-9]{1,100})(.csv)",
state_run)
- # print state_run
if match:
state_run_iterations.append(float(match.group(2)))
- # print state_run_iterations
run_and_results_fn = state_runs[np.argmax(state_run_iterations)]
- # print run_and_results_fn
runs_and_results = open(run_and_results_fn)
lines = runs_and_results.readlines()
@@ -187,13 +203,15 @@ def main(config, options, experiment_dir, **kwargs):
# **kwargs: Nothing so far
time_string = wrapping_util.get_time_string()
+ optimizer_str = os.path.splitext(os.path.basename(__file__))[0]
+
# Find experiment directory
if options.restore:
if not os.path.exists(options.restore):
raise Exception("The restore directory does not exist")
optimizer_dir = options.restore
else:
- optimizer_dir = os.path.join(experiment_dir, "smac_" +
+ optimizer_dir = os.path.join(experiment_dir, optimizer_str + "_" +
str(options.seed) + "_" + time_string)
# Build call
cmd = build_smac_call(config, options, optimizer_dir)
@@ -207,12 +225,12 @@ def main(config, options, experiment_dir, **kwargs):
# Copy the smac search space and create the instance information
fh = open(os.path.join(optimizer_dir, 'train.txt'), "w")
- for i in range(config.getint('DEFAULT', 'numberCV')):
+ for i in range(config.getint('HPOLIB', 'number_cv_folds')):
fh.write(str(i) + "\n")
fh.close()
fh = open(os.path.join(optimizer_dir, 'test.txt'), "w")
- for i in range(config.getint('DEFAULT', 'numberCV')):
+ for i in range(config.getint('HPOLIB', 'number_cv_folds')):
fh.write(str(i) + "\n")
fh.close()
@@ -220,12 +238,13 @@ def main(config, options, experiment_dir, **kwargs):
fh.close()
if not os.path.exists(os.path.join(optimizer_dir, params)):
- os.symlink(os.path.join(experiment_dir, "smac", params),
+ os.symlink(os.path.join(experiment_dir, optimizer_str, params),
os.path.join(optimizer_dir, params))
- sys.stdout.write("### INFORMATION ################################################################\n")
- sys.stdout.write("# You're running %40s #\n" % path_to_optimizer)
- sys.stdout.write("%s" % version_info)
- sys.stdout.write("# A newer version might be available, but not yet built in. #\n")
- sys.stdout.write("# Please use this version only to reproduce our results on automl.org #\n")
- sys.stdout.write("################################################################################\n")
+ logger.info("### INFORMATION ################################################################")
+ logger.info("# You're running %40s #" % config.get('SMAC', 'path_to_optimizer'))
+ for v in version_info:
+ logger.info("# %76s #" % v)
+ logger.info("# A newer version might be available, but not yet built in. #")
+ logger.info("# Please use this version only to reproduce our results on automl.org #")
+ logger.info("################################################################################")
return cmd, optimizer_dir
\ No newline at end of file
diff --git a/optimizers/smac/smac_2_06_01-devDefault.cfg b/optimizers/smac/smac_2_06_01-devDefault.cfg
new file mode 100644
index 00000000..c0d70c79
--- /dev/null
+++ b/optimizers/smac/smac_2_06_01-devDefault.cfg
@@ -0,0 +1,37 @@
+[HPOLIB]
+# Because smac does intensify and does not need to run all k-folds
+handles_cv=1
+
+[SMAC]
+# cutoff_time = runsolver_time_limit + 100 sec
+
+# Set otherwise
+# algo_exec = %(run_instance)s
+# num_concurrent_algo_execs = %(number_of_concurrent_jobs)s
+# experimentDir = %(exp_directory)s/
+# outputDirectory = %(exp_directory)s/smac/output/
+# total_num_runs_limit = %(number_of_jobs)s * %(cv)s
+
+num_run = 0
+intra_instance_obj = MEAN
+run_obj = QUALITY
+#instance_file = train.txt
+#test_instance_file = test.txt
+#execDir = ./
+p = smac_2_06_01-dev/params.pcs
+rf_full_tree_bootstrap = False
+rf_split_min = 10
+
+adaptive_capping = false
+max_incumbent_runs = 2000
+num_iterations = 2147483647
+# No one actually cares about it...
+runtime_limit = 2147483647
+
+deterministic = True
+retry_target_algorithm_run_count = 0
+intensification_percentage = 0
+validation = false
+
+# either relative to __file__ or absolute
+path_to_optimizer = ./smac_2_06_01-dev_src
diff --git a/optimizers/smac/smac_2_06_01-dev_parser.py b/optimizers/smac/smac_2_06_01-dev_parser.py
new file mode 100644
index 00000000..b8e5a5c3
--- /dev/null
+++ b/optimizers/smac/smac_2_06_01-dev_parser.py
@@ -0,0 +1,59 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import logging
+import os
+import sys
+
+import ConfigParser
+
+logger = logging.getLogger("HPOlib.optimizers.smac.smac_2_06_01-dev_parser")
+
+
+def manipulate_config(config):
+ if not config.has_option('SMAC', 'cutoff_time'):
+ print config.get('HPOLIB', 'runsolver_time_limit')
+ if config.get('HPOLIB', 'runsolver_time_limit'):
+ config.set('SMAC', 'cutoff_time',
+ str(config.getint('HPOLIB', 'runsolver_time_limit') + 100))
+ else:
+ # SMACs maxint
+ config.set('SMAC', 'cutoff_time', "2147483647")
+ if not config.has_option('SMAC', 'algo_exec'):
+ config.set('SMAC', 'algo_exec',
+ config.get('HPOLIB', 'run_instance'))
+ if not config.has_option('SMAC', 'total_num_runs_limit'):
+ config.set('SMAC', 'total_num_runs_limit',
+ str(config.getint('HPOLIB', 'number_of_jobs') *
+ config.getint('HPOLIB', 'number_cv_folds')))
+ if not config.has_option('SMAC', 'num_concurrent_algo_execs'):
+ config.set('SMAC', 'num_concurrent_algo_execs',
+ config.get('HPOLIB', 'number_of_concurrent_jobs'))
+
+ path_to_optimizer = config.get('SMAC', 'path_to_optimizer')
+ if not os.path.isabs(path_to_optimizer):
+ path_to_optimizer = os.path.join(os.path.dirname(os.path.realpath(__file__)), path_to_optimizer)
+
+ path_to_optimizer = os.path.normpath(path_to_optimizer)
+ if not os.path.exists(path_to_optimizer):
+ logger.critical("Path to optimizer not found: %s" % path_to_optimizer)
+ sys.exit(1)
+
+ config.set('SMAC', 'path_to_optimizer', path_to_optimizer)
+
+ return config
\ No newline at end of file
diff --git a/spearmint.py b/optimizers/spearmint/spearmint_april2013_mod.py
similarity index 64%
rename from spearmint.py
rename to optimizers/spearmint/spearmint_april2013_mod.py
index 8d739ba5..cc52564d 100644
--- a/spearmint.py
+++ b/optimizers/spearmint/spearmint_april2013_mod.py
@@ -16,30 +16,52 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-#!/usr/bin/env python
-
import cPickle
+import logging
import os
import sys
import numpy as np
-import wrapping_util
+import HPOlib.wrapping_util
+
__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
__contact__ = "automl.org"
+
+logger = logging.getLogger("HPOlib.spearmint_april2013_mod")
+
+
path_to_optimizer = "optimizers/spearmint_april2013_mod/"
version_info = ("# %76s #\n" %
"https://github.com/JasperSnoek/spearmint/tree/613350f2f617de3af5f101b1dc5eccf60867f67e")
+def check_dependencies():
+ try:
+ import google.protobuf
+ except ImportError:
+ raise ImportError("Google protobuf cannot be imported. Are you sure "
+ "it's installed?")
+ try:
+ import numpy
+ except ImportError:
+ raise ImportError("Numpy cannot be imported. Are you sure that it's"
+ " installed?")
+ try:
+ import scipy
+ except ImportError:
+ raise ImportError("Scipy cannot be imported. Are you sure that it's"
+ " installed?")
+
+
def build_spearmint_call(config, options, optimizer_dir):
- call = 'python ' + os.path.dirname(os.path.realpath(__file__)) + \
- "/" + path_to_optimizer + '/spearmint_sync.py'
+ print
+ call = 'python ' + os.path.join(config.get('SPEARMINT', 'path_to_optimizer'), 'spearmint_sync.py')
call = ' '.join([call, optimizer_dir,
'--config', config.get('SPEARMINT', 'config'),
- '--max-concurrent', config.get('DEFAULT', 'numberOfConcurrentJobs'),
+ '--max-concurrent', config.get('HPOLIB', 'number_of_concurrent_jobs'),
'--max-finished-jobs', config.get('SPEARMINT', 'max_finished_jobs'),
'--polling-time', config.get('SPEARMINT', 'spearmint_polling_time'),
'--grid-size', config.get('SPEARMINT', 'grid_size'),
@@ -48,7 +70,7 @@ def build_spearmint_call(config, options, optimizer_dir):
'--grid-seed', str(options.seed)])
if config.get('SPEARMINT', 'method') != "GPEIChooser" and \
config.get('SPEARMINT', 'method') != "GPEIOptChooser":
- sys.stdout.write('WARNING: This chooser might not work yet\n')
+ logger.warning('WARNING: This chooser might not work yet\n')
call = ' '.join([call, config.get("SPEARMINT", 'method_args')])
return call
@@ -61,17 +83,18 @@ def restore(config, optimizer_dir, **kwargs):
"""
restore_file = os.path.join(optimizer_dir, "expt-grid.pkl")
if not os.path.exists(restore_file):
- print "Oups, this should have been checked before"
+ logger.error("Oups, this should have been checked before")
raise Exception("%s does not exist" % (restore_file,))
sys.path.append(os.path.join(
os.path.dirname(os.path.realpath(__file__)), path_to_optimizer))
+ # We need the Grid because otherwise we cannot load the pickle file
import ExperimentGrid
# Assumes that all not valid states are marked as crashed
fh = open(restore_file)
exp_grid = cPickle.load(fh)
fh.close()
complete_runs = np.sum(exp_grid['status'] == 3)
- restored_runs = complete_runs * config.getint('DEFAULT', 'numberCV')
+ restored_runs = complete_runs * config.getint('HPOLIB', 'number_cv_folds')
try:
os.remove(os.path.join(optimizer_dir, "expt-grid.pkl.lock"))
except OSError:
@@ -86,7 +109,8 @@ def main(config, options, experiment_dir, **kwargs):
# experiment_dir: Experiment directory/Benchmark_directory
# **kwargs: Nothing so far
- time_string = wrapping_util.get_time_string()
+ time_string = HPOlib.wrapping_util.get_time_string()
+ optimizer_str = os.path.splitext(os.path.basename(__file__))[0]
# Find experiment directory
if options.restore:
@@ -94,8 +118,8 @@ def main(config, options, experiment_dir, **kwargs):
raise Exception("The restore directory does not exist")
optimizer_dir = options.restore
else:
- optimizer_dir = os.path.join(experiment_dir, "spearmint_" \
- + str(options.seed) + "_" + time_string)
+ optimizer_dir = os.path.join(experiment_dir, optimizer_str + "_" +
+ str(options.seed) + "_" + time_string)
# Build call
cmd = build_spearmint_call(config, options, optimizer_dir)
@@ -106,12 +130,12 @@ def main(config, options, experiment_dir, **kwargs):
# Make a link to the Protocol-Buffer config file
configpb = config.get('SPEARMINT', 'config')
if not os.path.exists(os.path.join(optimizer_dir, configpb)):
- os.symlink(os.path.join(experiment_dir, "spearmint", configpb),
+ os.symlink(os.path.join(experiment_dir, optimizer_str, configpb),
os.path.join(optimizer_dir, configpb))
- sys.stdout.write("### INFORMATION ################################################################\n")
- sys.stdout.write("# You're running %40s #\n" % path_to_optimizer)
- sys.stdout.write("%s" % version_info)
- sys.stdout.write("# A newer version might be available, but not yet built in. #\n")
- sys.stdout.write("# Please use this version only to reproduce our results on automl.org #\n")
- sys.stdout.write("################################################################################\n")
+ logger.info("### INFORMATION ################################################################")
+ logger.info("# You're running %40s #" % path_to_optimizer)
+ logger.info("%s" % version_info)
+ logger.info("# A newer version might be available, but not yet built in. #")
+ logger.info("# Please use this version only to reproduce our results on automl.org #")
+ logger.info("################################################################################")
return cmd, optimizer_dir
\ No newline at end of file
diff --git a/config_parser/spearmintDefault.cfg b/optimizers/spearmint/spearmint_april2013_modDefault.cfg
similarity index 82%
rename from config_parser/spearmintDefault.cfg
rename to optimizers/spearmint/spearmint_april2013_modDefault.cfg
index 659d5807..eed72c5a 100644
--- a/config_parser/spearmintDefault.cfg
+++ b/optimizers/spearmint/spearmint_april2013_modDefault.cfg
@@ -18,4 +18,7 @@ wrapper = 0
spearmint_polling_time = 3.0
#General Parameters
max_concurrent = 1
-#max_finished_jobs = %(numberOfJobs)s
\ No newline at end of file
+#max_finished_jobs = %(number_of_jobs)s
+
+# either relative to __file__ or absolute
+path_to_optimizer = ./spearmint_april2013_mod_src
diff --git a/optimizers/spearmint/spearmint_april2013_mod_parser.py b/optimizers/spearmint/spearmint_april2013_mod_parser.py
new file mode 100644
index 00000000..85df4d02
--- /dev/null
+++ b/optimizers/spearmint/spearmint_april2013_mod_parser.py
@@ -0,0 +1,49 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import logging
+import os
+import sys
+
+logger = logging.getLogger("HPOlib.optimizers.spearmint.spearmint_april2013_mod_parser")
+
+
+def manipulate_config(config):
+ # special cases
+ if not config.has_option('SPEARMINT', 'method'):
+ raise Exception("SPEARMINT:method not specified in .cfg")
+ if not config.has_option('SPEARMINT', 'method_args'):
+ raise Exception("SPEARMINT:method-args not specified in .cfg")
+
+ # GENERAL
+ if not config.has_option('SPEARMINT', 'max_finished_jobs'):
+ config.set('SPEARMINT', 'max_finished_jobs',
+ config.get('HPOLIB', 'number_of_jobs'))
+
+ path_to_optimizer = config.get('SPEARMINT', 'path_to_optimizer')
+ if not os.path.isabs(path_to_optimizer):
+ path_to_optimizer = os.path.join(os.path.dirname(os.path.realpath(__file__)), path_to_optimizer)
+
+ path_to_optimizer = os.path.normpath(path_to_optimizer)
+ if not os.path.exists(path_to_optimizer):
+ logger.critical("Path to optimizer not found: %s" % path_to_optimizer)
+ sys.exit(1)
+
+ config.set('SPEARMINT', 'path_to_optimizer', path_to_optimizer)
+
+ return config
diff --git a/optimizers/tpe/hyperopt_august2013_mod.py b/optimizers/tpe/hyperopt_august2013_mod.py
new file mode 100644
index 00000000..bc5ee359
--- /dev/null
+++ b/optimizers/tpe/hyperopt_august2013_mod.py
@@ -0,0 +1,178 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import cPickle
+import logging
+import os
+import sys
+
+import HPOlib.wrapping_util as wrapping_util
+
+logger = logging.getLogger("HPOlib.optimizers.tpe.hyperopt_august2013_mod")
+
+version_info = ("# %76s #" % "https://github.com/hyperopt/hyperopt/tree/486aebec8a4170e4781d99bbd6cca09123b12717")
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+# noinspection PyUnresolvedReferences
+def check_dependencies():
+ try:
+ import nose
+ logger.debug("\tNose: %s\n" % str(nose.__version__))
+ except ImportError:
+ raise ImportError("Nose cannot be imported. Are you sure it's "
+ "installed?")
+ try:
+ import networkx
+ logger.debug("\tnetworkx: %s\n" % str(networkx.__version__))
+ except ImportError:
+ raise ImportError("Networkx cannot be imported. Are you sure it's "
+ "installed?")
+ try:
+ import pymongo
+ logger.debug("\tpymongo: %s\n" % str(pymongo.version))
+ from bson.objectid import ObjectId
+ except ImportError:
+ raise ImportError("Pymongo cannot be imported. Are you sure it's"
+ " installed?")
+ try:
+ import numpy
+ logger.debug("\tnumpy: %s" % str(numpy.__version__))
+ except ImportError:
+ raise ImportError("Numpy cannot be imported. Are you sure that it's"
+ " installed?")
+ try:
+ import scipy
+ logger.debug("\tscipy: %s" % str(scipy.__version__))
+ except ImportError:
+ raise ImportError("Scipy cannot be imported. Are you sure that it's"
+ " installed?")
+
+
+def build_tpe_call(config, options, optimizer_dir):
+ # For TPE we have to cd to the exp_dir
+ call = "python " + os.path.dirname(os.path.realpath(__file__)) + \
+ "/tpecall.py"
+ call = ' '.join([call, '-p', config.get('TPE', 'space'),
+ "-m", config.get('TPE', 'number_evals'),
+ "-s", str(options.seed),
+ "--cwd", optimizer_dir])
+ if options.restore:
+ call = ' '.join([call, '-r'])
+ return call
+
+
+#noinspection PyUnusedLocal
+def restore(config, optimizer_dir, **kwargs):
+ """
+ Returns the number of restored runs. This is the number of different configs
+ tested multiplied by the number of crossvalidation folds.
+ """
+ restore_file = os.path.join(optimizer_dir, 'state.pkl')
+ if not os.path.exists(restore_file):
+ print "Oups, this should have been checked before"
+ raise Exception("%s does not exist" % (restore_file,))
+
+ # Special settings for restoring
+ fh = open(restore_file)
+ state = cPickle.load(fh)
+ fh.close()
+ complete_runs = 0
+ #noinspection PyProtectedMember
+ tpe_trials = state['trials']._trials
+ for trial in tpe_trials:
+ # Assumes that all not valid states states are marked crashed
+ if trial['state'] == 2:
+ complete_runs += 1
+ restored_runs = complete_runs * config.getint('HPOLIB', 'number_cv_folds')
+ return restored_runs
+
+
+#noinspection PyUnusedLocal
+def main(config, options, experiment_dir, **kwargs):
+ # config: Loaded .cfg file
+ # options: Options containing seed, restore_dir,
+ # experiment_dir: Experiment directory/Benchmark_directory
+ # **kwargs: Nothing so far
+ time_string = wrapping_util.get_time_string()
+ cmd = ""
+
+ # Add path_to_optimizer to PYTHONPATH and to sys.path
+ # Only for HYPEROPT
+ if not 'PYTHONPATH' in os.environ:
+ os.environ['PYTHONPATH'] = config.get('TPE', 'path_to_optimizer')
+ else:
+ os.environ['PYTHONPATH'] = config.get('TPE', 'path_to_optimizer') + os.pathsep + os.environ['PYTHONPATH']
+ sys.path.append(config.get('TPE', 'path_to_optimizer'))
+
+ optimizer_str = os.path.splitext(os.path.basename(__file__))[0]
+
+# TODO: Check whether we might need this again
+# SYSTEM_WIDE = 0
+# AUGUST_2013_MOD = 1
+# try:
+# import hyperopt
+# version = SYSTEM_WIDE
+# except ImportError:
+# try:
+# cmd += "export PYTHONPATH=$PYTHONPATH:" + os.path.dirname(os.path.abspath(__file__)) + \
+# "/optimizers/hyperopt_august2013_mod\n"
+# import optimizers.hyperopt_august2013_mod.hyperopt as hyperopt
+# except ImportError, e:
+# import HPOlib.optimizers.hyperopt_august2013_mod.hyperopt as hyperopt
+# version = AUGUST_2013_MOD
+
+ # Find experiment directory
+ if options.restore:
+ if not os.path.exists(options.restore):
+ raise Exception("The restore directory does not exist")
+ optimizer_dir = options.restore
+ else:
+ optimizer_dir = os.path.join(experiment_dir, optimizer_str + "_" +
+ str(options.seed) + "_" +
+ time_string)
+
+ # Build call
+ cmd += build_tpe_call(config, options, optimizer_dir)
+
+ # Set up experiment directory
+ if not os.path.exists(optimizer_dir):
+ os.mkdir(optimizer_dir)
+ space = config.get('TPE', 'space')
+ # Copy the hyperopt search space
+ if not os.path.exists(os.path.join(optimizer_dir, space)):
+ os.symlink(os.path.join(experiment_dir, optimizer_str, space),
+ os.path.join(optimizer_dir, space))
+
+ import hyperopt
+ path_to_loaded_optimizer = os.path.abspath(os.path.dirname(os.path.dirname(hyperopt.__file__)))
+
+ logger.info("### INFORMATION ################################################################")
+ logger.info("# You are running: #")
+ logger.info("# %76s #" % path_to_loaded_optimizer)
+ if not os.path.samefile(path_to_loaded_optimizer, config.get('TPE', 'path_to_optimizer')):
+ logger.warning("# BUT hyperopt_august2013_modDefault.cfg says:")
+ logger.warning("# %76s #" % config.get('TPE', 'path_to_optimizer'))
+ logger.warning("# Found a global hyperopt version. This installation will be used! #")
+ else:
+ logger.info("# To reproduce our results you need version 0.0.3.dev, which can be found here:#")
+ logger.info("%s" % version_info)
+ logger.info("# A newer version might be available, but not yet built in. #")
+ logger.info("################################################################################")
+ return cmd, optimizer_dir
diff --git a/optimizers/tpe/hyperopt_august2013_modDefault.cfg b/optimizers/tpe/hyperopt_august2013_modDefault.cfg
new file mode 100644
index 00000000..09cde416
--- /dev/null
+++ b/optimizers/tpe/hyperopt_august2013_modDefault.cfg
@@ -0,0 +1,7 @@
+[TPE]
+space = space.py
+#exp_dir = %(exp_directory)s/tpe/
+#number_evals = %(number_of_jobs)s
+
+# either relative to __file__ or absolute
+path_to_optimizer = ./hyperopt_august2013_mod_src
\ No newline at end of file
diff --git a/optimizers/tpe/hyperopt_august2013_mod_parser.py b/optimizers/tpe/hyperopt_august2013_mod_parser.py
new file mode 100644
index 00000000..418c40b7
--- /dev/null
+++ b/optimizers/tpe/hyperopt_august2013_mod_parser.py
@@ -0,0 +1,50 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import logging
+import os
+import sys
+
+import ConfigParser
+
+logger = logging.getLogger("HPOlib.optimizers.tpe.hyperopt_august2013_mod_parser")
+
+
+def manipulate_config(config):
+ if not config.has_section('TPE'):
+ config.add_section('TPE')
+
+ # optional cases
+ if not config.has_option('TPE', 'space'):
+ raise Exception("TPE:space not specified in .cfg")
+
+ if not config.has_option('TPE', 'number_evals'):
+ config.set('TPE', 'number_evals', config.get('HPOLIB', 'number_of_jobs'))
+
+ path_to_optimizer = config.get('TPE', 'path_to_optimizer')
+ if not os.path.isabs(path_to_optimizer):
+ path_to_optimizer = os.path.join(os.path.dirname(os.path.realpath(__file__)), path_to_optimizer)
+
+ path_to_optimizer = os.path.normpath(path_to_optimizer)
+ if not os.path.exists(path_to_optimizer):
+ logger.critical("Path to optimizer not found: %s" % path_to_optimizer)
+ sys.exit(1)
+
+ config.set('TPE', 'path_to_optimizer', path_to_optimizer)
+
+ return config
\ No newline at end of file
diff --git a/optimizers/tpe/random_hyperopt_august2013_mod.py b/optimizers/tpe/random_hyperopt_august2013_mod.py
new file mode 100644
index 00000000..1d35ebcf
--- /dev/null
+++ b/optimizers/tpe/random_hyperopt_august2013_mod.py
@@ -0,0 +1,161 @@
+#!/usr/bin/env python
+
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import cPickle
+import logging
+import os
+import sys
+
+import HPOlib.wrapping_util as wrappingUtil
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+logger = logging.getLogger("HPOlib.optimizers.tpe.randomtpe")
+
+version_info = ("# %76s #" % "https://github.com/hyperopt/hyperopt/tree/486aebec8a4170e4781d99bbd6cca09123b12717")
+
+
+# noinspection PyUnresolvedReferences
+def check_dependencies():
+ try:
+ import nose
+ logger.debug("\tNose: %s\n" % str(nose.__version__))
+ except ImportError:
+ raise ImportError("Nose cannot be imported. Are you sure it's "
+ "installed?")
+ try:
+ import networkx
+ logger.debug("\tnetworkx: %s\n" % str(networkx.__version__))
+ except ImportError:
+ raise ImportError("Networkx cannot be imported. Are you sure it's "
+ "installed?")
+ try:
+ import pymongo
+ logger.debug("\tpymongo: %s\n" % str(pymongo.version))
+ from bson.objectid import ObjectId
+ except ImportError:
+ raise ImportError("Pymongo cannot be imported. Are you sure it's"
+ " installed?")
+ try:
+ import numpy
+ logger.debug("\tnumpy: %s" % str(numpy.__version__))
+ except ImportError:
+ raise ImportError("Numpy cannot be imported. Are you sure that it's"
+ " installed?")
+ try:
+ import scipy
+ logger.debug("\tscipy: %s" % str(scipy.__version__))
+ except ImportError:
+ raise ImportError("Scipy cannot be imported. Are you sure that it's"
+ " installed?")
+
+
+
+def build_random_call(config, options, optimizer_dir):
+ call = "python " + os.path.dirname(os.path.realpath(__file__)) + \
+ "/tpecall.py"
+ call = ' '.join([call, '-p', config.get('TPE', 'space'),
+ "-m", config.get('TPE', 'number_evals'),
+ "-s", str(options.seed),
+ "--cwd", optimizer_dir, "--random"])
+ if options.restore:
+ call = ' '.join([call, '-r'])
+ return call
+
+
+# noinspection PyUnusedLocal
+def restore(config, optimizer_dir, **kwargs):
+ restore_file = os.path.join(optimizer_dir, 'state.pkl')
+ if not os.path.exists(restore_file):
+ logger.error("Oups, this should have been checked before")
+ raise Exception("%s does not exist" % (restore_file,))
+
+ fh = open(restore_file)
+ state = cPickle.load(fh)
+ fh.close()
+ complete_runs = 0
+ # noinspection PyProtectedMember
+ tpe_trials = state['trials']._trials
+ for trial in tpe_trials:
+ # Assumes that all states no valid state is marked crashed
+ if trial['state'] == 2:
+ complete_runs += 1
+ restored_runs = complete_runs * config.getint('HPOLIB', 'number_cv_folds')
+ return restored_runs
+
+
+# noinspection PyUnusedLocal
+def main(config, options, experiment_dir, **kwargs):
+ # config: Loaded .cfg file
+ # options: Options containing seed, restore,
+ # experiment_dir: Experiment directory/Benchmarkdirectory
+ # **kwargs: Nothing so far
+ time_string = wrappingUtil.get_time_string()
+ cmd = ""
+
+ # Add path_to_optimizer to PYTHONPATH and to sys.path
+ if not 'PYTHONPATH' in os.environ:
+ os.environ['PYTHONPATH'] = config.get('TPE', 'path_to_optimizer')
+ else:
+ os.environ['PYTHONPATH'] = config.get('TPE', 'path_to_optimizer') + os.pathsep + os.environ['PYTHONPATH']
+ sys.path.append(config.get('TPE', 'path_to_optimizer'))
+ optimizer_str = os.path.splitext(os.path.basename(__file__))[0]
+
+ # Find experiment directory
+ if options.restore:
+ if not os.path.exists(options.restore):
+ raise Exception("The restore directory does not exist")
+ optimizer_dir = options.restore
+ else:
+ optimizer_dir = os.path.join(experiment_dir, optimizer_str + "_" +
+ str(options.seed) + "_" +
+ time_string)
+
+ # Build call
+ cmd = build_random_call(config, options, optimizer_dir)
+
+ # Set up experiment directory
+ if not os.path.exists(optimizer_dir):
+ os.mkdir(optimizer_dir)
+ space = config.get('TPE', 'space')
+ # Copy the hyperopt search space
+ if not os.path.exists(os.path.join(optimizer_dir, space)):
+ os.symlink(os.path.join(experiment_dir, optimizer_str, space),
+ os.path.join(optimizer_dir, space))
+
+ import hyperopt
+ path_to_loaded_optimizer = os.path.abspath(os.path.dirname(os.path.dirname(hyperopt.__file__)))
+
+ logger.info("### INFORMATION ################################################################")
+ logger.info("# You are running: #")
+ logger.info("# %76s #" % path_to_loaded_optimizer)
+ if not os.path.samefile(path_to_loaded_optimizer, config.get('TPE', 'path_to_optimizer')):
+ logger.warning("# BUT random_hyperopt_august2013_modDefault.cfg says:")
+ logger.warning("# %76s #" % config.get('TPE', 'path_to_optimizer'))
+ logger.warning("# Found a global hyperopt version. This installation will be used! #")
+ else:
+ logger.info("# To reproduce our results you need version 0.0.3.dev, which can be found here:#")
+ logger.info("%s" % version_info)
+ logger.info("# A newer version might be available, but not yet built in. #")
+ logger.info("################################################################################")
+
+ return cmd, optimizer_dir
diff --git a/optimizers/tpe/random_hyperopt_august2013_modDefault.cfg b/optimizers/tpe/random_hyperopt_august2013_modDefault.cfg
new file mode 120000
index 00000000..a3a72127
--- /dev/null
+++ b/optimizers/tpe/random_hyperopt_august2013_modDefault.cfg
@@ -0,0 +1 @@
+hyperopt_august2013_modDefault.cfg
\ No newline at end of file
diff --git a/optimizers/tpe/random_hyperopt_august2013_mod_parser.py b/optimizers/tpe/random_hyperopt_august2013_mod_parser.py
new file mode 120000
index 00000000..f32f95f7
--- /dev/null
+++ b/optimizers/tpe/random_hyperopt_august2013_mod_parser.py
@@ -0,0 +1 @@
+hyperopt_august2013_mod_parser.py
\ No newline at end of file
diff --git a/tpecall.py b/optimizers/tpe/tpecall.py
similarity index 64%
rename from tpecall.py
rename to optimizers/tpe/tpecall.py
index 02cbeb1e..0bcaf632 100644
--- a/tpecall.py
+++ b/optimizers/tpe/tpecall.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
##
# wrapping: A program making it easy to use hyperparameter
# optimization software.
@@ -15,17 +17,19 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-
-#!/usr/bin/env python
+from argparse import ArgumentParser
import cPickle
-from optparse import OptionParser
from functools import partial
from importlib import import_module
+import logging
import os
-import numpy as np
+import sys
import hyperopt
+import HPOlib.cv as cv
+
+logger = logging.getLogger("HPOlib.optimizers.tpe.tpecall")
__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
__contact__ = "automl.org"
@@ -56,56 +60,55 @@ def pyll_replace_list_with_dict(search_space, indent = 0):
def main():
- # Parse options and arguments
- parser = OptionParser()
- parser.add_option("-p", "--space",
- dest="spaceFile",
- help="Where is the space.py located?")
- parser.add_option("-a", "--algoExec",
- dest="algoExec",
- help="Which function to load located?")
- parser.add_option("-m", "--maxEvals",
- dest="maxEvals",
- help="How many evaluations?")
- parser.add_option("-s", "--seed",
- dest="seed",
- default="123",
- type=int,
- help="Seed for the TPE algorithm")
- parser.add_option("-r", "--restore",
- dest="restore",
- action="store_true",
- help="When this flag is set state.pkl is restored in " +
+ prog = "python statistics.py WhatIsThis WhatIsThis [WhatIsThis ]"
+ description = "Return some statistical information"
+
+ parser = ArgumentParser(description=description, prog=prog)
+
+ parser.add_argument("-p", "--space",
+ dest="spaceFile", help="Where is the space.py located?")
+ parser.add_argument("-m", "--maxEvals",
+ dest="maxEvals", help="How many evaluations?")
+ parser.add_argument("-s", "--seed", default="1",
+ dest="seed", type=int, help="Seed for the TPE algorithm")
+ parser.add_argument("-r", "--restore", action="store_true",
+ dest="restore", help="When this flag is set state.pkl is restored in " +
"the current working directory")
- parser.add_option("--random", default=False,
- dest="random",
- action="store_true",
- help="Use a random search")
- (options, args) = parser.parse_args()
+ parser.add_argument("--random", default=False, action="store_true",
+ dest="random", help="Use a random search")
+ parser.add_argument("--cwd", help="Change the working directory before "
+ "optimizing.")
+
+ args, unknown = parser.parse_known_args()
+
+ if args.cwd:
+ os.chdir(args.cwd)
+
+ if not os.path.exists(args.spaceFile):
+ logger.critical("Search space not found: %s" % args.spaceFile)
+ sys.exit(1)
# First remove ".py"
- algo, ext = os.path.splitext(os.path.basename(options.algoExec))
- space, ext = os.path.splitext(os.path.basename(options.spaceFile))
+ space, ext = os.path.splitext(os.path.basename(args.spaceFile))
# Then load dict searchSpace and out function cv.py
- import sys
sys.path.append("./")
sys.path.append("")
- print os.getcwd()
+
module = import_module(space)
search_space = module.space
- fn = import_module(algo)
- fn = fn.doForTPE
+ fn = cv.main # doForTPE
- if options.random:
+ if args.random:
# We use a random search
- tpe_with_seed = partial(hyperopt.tpe.rand.suggest, seed=int(options.seed))
+ tpe_with_seed = partial(hyperopt.tpe.rand.suggest, seed=int(args.seed))
+ logger.info("Using Random Search")
else:
- tpe_with_seed = partial(hyperopt.tpe.suggest, seed=int(options.seed))
+ tpe_with_seed = partial(hyperopt.tpe.suggest, seed=int(args.seed))
# Now run TPE, emulate fmin.fmin()
state_filename = "state.pkl"
- if options.restore:
+ if args.restore:
# We do not need to care about the state of the trials object since it
# is only serialized in a synchronized state, there will never be a save
# with a running experiment
@@ -115,14 +118,14 @@ def main():
trials = tmp_dict['trials']
print trials.__dict__
else:
- domain = hyperopt.Domain(fn, search_space, rseed=int(options.seed))
+ domain = hyperopt.Domain(fn, search_space, rseed=int(args.seed))
trials = hyperopt.Trials()
fh = open(state_filename, "w")
# By this we probably loose the seed; not too critical for a restart
cPickle.dump({"trials": trials, "domain": domain}, fh)
fh.close()
- for i in range(int(options.maxEvals) + 1):
+ for i in range(int(args.maxEvals) + 1):
# in exhaust, the number of evaluations is max_evals - num_done
rval = hyperopt.FMinIter(tpe_with_seed, domain, trials, max_evals=i)
rval.exhaust()
diff --git a/randomtpe.py b/randomtpe.py
deleted file mode 100644
index 3243ad10..00000000
--- a/randomtpe.py
+++ /dev/null
@@ -1,93 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-#!/usr/bin/env python
-
-import cPickle
-import os
-
-from config_parser.parse import parse_config
-import wrapping_util
-
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__contact__ = "automl.org"
-
-
-def buildRandomCall(config, options, optimizer_dir):
- #For TPE (and Random Search) we have to cd to the exp_dir
- call = 'cd ' + optimizer_dir + '\n' + \
- 'python ' + os.path.dirname(os.path.realpath(__file__)) + \
- '/tpecall.py'
- call = ' '.join([call, '-p', config.get('TPE', 'space'), \
- '-a', config.get('DEFAULT', 'algorithm'), \
- '-m', config.get('DEFAULT', 'numberOfJobs'), \
- '-s', str(options.seed), '--random'])
- if options.restore:
- call = ' '.join([call, '-r'])
- return call
-
-
-def restore(config, optimizer_dir, **kwargs):
- restore_file = os.path.join(optimizer_dir, 'state.pkl')
- if not os.path.exists(restore_file):
- print "Oups, this should have been checked before"
- raise Exception("%s does not exist" % (restore_file,))
- return -1
-
- fh = open(restore_file)
- state = cPickle.load(fh)
- fh.close()
- complete_runs = 0
- tpe_trials = state['trials']._trials
- for trial in tpe_trials:
- # Assumes that all states no valid state is marked crashed
- if trial['state'] == 2:
- complete_runs += 1
- restored_runs = complete_runs * config.getint('DEFAULT', 'numberCV')
- return restored_runs
-
-
-def main(config, options, experiment_dir, **kwargs):
- # config: Loaded .cfg file
- # options: Options containing seed, restore,
- # experiment_dir: Experiment directory/Benchmarkdirectory
- # **kwargs: Nothing so far
- time_string = wrapping_util.get_time_string()
-
- # Find experiment directory
- if options.restore:
- if not os.path.exists(options.restore):
- raise Exception("The restore directory %s does not exist" % (options.restore,))
- optimizer_dir = options.restore
- else:
- optimizer_dir = os.path.join(experiment_dir, "randomtpe_" + \
- str(options.seed) + "_" + time_string)
-
- # Build call
- cmd = buildRandomCall(config, options, optimizer_dir)
-
- # Set up experiment directory
- if not os.path.exists(optimizer_dir):
- os.mkdir(optimizer_dir)
- space = config.get('TPE', 'space')
- # Copy the hyperopt search space
- if not os.path.exists(os.path.join(optimizer_dir, space)):
- os.symlink(os.path.join(experiment_dir, "tpe", space),
- os.path.join(optimizer_dir, space))
-
- return cmd, optimizer_dir
\ No newline at end of file
diff --git a/run_instance.py b/run_instance.py
deleted file mode 100644
index ee577942..00000000
--- a/run_instance.py
+++ /dev/null
@@ -1,240 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-#!/usr/bin/env python
-
-import cPickle
-import imp
-from optparse import OptionParser
-import os
-import re
-import sys
-import time
-
-import numpy as np
-
-from config_parser.parse import parse_config
-import Experiment
-import wrapping_util
-
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__contact__ = "automl.org"
-
-
-def loadExperimentFile(pickle):
- experiment = Experiment.Experiment(os.path.split(pickle)[0],
- os.path.split(pickle)[1].split(".")[0])
- return experiment
-
-
-def remove_param_metadata(params):
- """
- Check whether some params are defined on the Log scale or with a Q value,
- must be marked with "LOG$_{paramname}" or Q[0-999]_$paramname
- LOG/Q will be removed from the paramname
- """
- for para in params:
- new_name = para
- if "LOG10_" in para:
- pos = para.find("LOG10")
- new_name = para[0:pos] + para[pos+6:]
- # new_name = new_name.strip("_")
- params[new_name] = np.power(10, float(params[para]))
- del params[para]
- elif "LOG2" in para:
- pos = para.find("LOG2_")
- new_name = para[0:pos] + para[pos+5:]
- # new_name = new_name.strip("_")
- params[new_name] = np.power(2, float(params[para]))
- del params[para]
- elif "LOG_" in para:
- pos = para.find("LOG")
- new_name = para[0:pos] + para[pos+4:]
- # new_name = new_name.strip("_")
- params[new_name] = np.exp(float(params[para]))
- del params[para]
- #Check for Q value, returns round(x/q)*q
- m = re.search(r'Q[0-999\.]{1,10}_', para)
- if m is not None:
- pos = new_name.find(m.group(0))
- tmp = new_name[0:pos] + new_name[pos+len(m.group(0)):]
- #tmp = tmp.strip("_")
- q = float(m.group(0)[1:-1])
- params[tmp] = round(float(params[new_name])/q)*q
- del params[new_name]
-
-
-def run_instance(fold, fn, params, cfg):
- # Run instance
- result = np.NaN
- status = "UNSAT"
- folds = cfg.getint('DEFAULT', 'numberCV')
-
- # Remove additional information about variables and change them accordingly
- remove_param_metadata(params)
-
- starttime = time.time()
- try:
- result = fn(params, fold=fold, folds=folds)
- status = "SAT"
- except Exception:
- print wrapping_util.format_traceback(sys.exc_info())
- status = "CRASHED"
- duration = time.time() - starttime
- return result, duration, status
-
-
-def get_trial_index(experiment, fold, params):
- # Check whether we are in a new configuration; This has to check whether
- # the params were already inserted but also whether the fold already run
- # This is checked twice; the instance_result has to be not NaN and the
- # entry in instance_order has to exist
- new = True
- trial_index = np.NaN
- for idx, trial in enumerate(experiment.trials):
- exp = trial['params']
- if exp == params and (idx, fold) not in experiment.instance_order and \
- (experiment.get_trial_from_id(idx)['instance_results'][fold] == np.NaN or \
- experiment.get_trial_from_id(idx)['instance_results'][fold] !=
- experiment.get_trial_from_id(idx)['instance_results'][fold]):
- new = False
- trial_index = idx
- break
- if new:
- trial_index = experiment.add_job(params)
- return trial_index
-
-
-def main():
- # Parse options and arguments
- usage = "This script loads the module specified in the config.cfg" + \
- "file as 'function'.\nIt will either run the main() method or the"+\
- " run_test() method and print the output as: \n" + \
- "'Result for ParamILS: , , , " + \
- ", , ' \n\n" + \
- "%prog [-f -c | -d] \n" + \
- "where is a pickled dict of parameters\n"
- parser = OptionParser(usage=usage)
- parser.add_option("-f", "--fold",
- action="store", dest="fold", type="int", default=None,
- help="Number of fold to execute")
- parser.add_option("-c", "--config",
- action="store", dest="configFile", default=None,
- help="path to the config.cfg")
- parser.add_option("-s", "--seed",
- action="store", dest="seed", type="int", default=1,
- help="Seed to use for calculations")
- parser.add_option("-p", "--pkl",
- action="store", dest="pkl", default=None,
- help="Pickle file for bookkeeping")
- parser.add_option("-t", "--test",
- action = "store_true", dest = "test", default = False,
- help = "With this flag activated a function run_test " + \
- "is called instead of main")
- (options, args) = parser.parse_args()
-
- if (options.configFile is None and options.fold is not None) or \
- (options.configFile is not None and options.fold is None):
- parser.error("If not using -d, fold and config.cfg need " + \
- "to be specified")
- if options.configFile is None and options.fold is None and \
- not options.debug:
- parser.print_usage()
-
- seed = options.seed
-
- # Load the pickled param file...
- if len(args) != 1:
- raise Exception("Too many or less paramfiles specified")
- paramfile = args[0]
- print "Paramfile", paramfile, os.path.exists(paramfile)
- if not os.path.exists(paramfile):
- raise Exception("%s is not a file\n" % paramfile)
- fh = open(paramfile, "r")
- params = cPickle.load(fh)
- fh.close()
-
- # ...then load config.cfg...
- cfg = parse_config(options.configFile, allow_no_value=True)
- if os.path.isabs(cfg.get("DEFAULT", "function")):
- fn_path = cfg.get("DEFAULT", "function")
- else:
- fn_path = cfg.get("DEFAULT", "function")
- fn_path_parent = os.path.join("..", cfg.get("DEFAULT", "function"))
- fn_name, ext = os.path.splitext(os.path.basename(fn_path))
- try:
- fn = imp.load_source(fn_name, fn_path)
- except (ImportError, IOError):
- try:
- fn = imp.load_source(fn_name, fn_path_parent)
- except (ImportError, IOError):
- print os.path.join(fn_path, "..")
- print(("Could not find\n%s\n\tin\n%s\n\tor its parent directory " +
- "relative to\n%s")
- % (fn_name, fn_path, os.getcwd()))
- import traceback
- print traceback.format_exc()
- sys.exit(1)
-
- if options.test:
- fn = fn.run_test
- else:
- fn = fn.main
-
- # Do bookkeeping before running the instance
- optimizer = "w/o bookkeeping"
-
- if options.pkl is not None:
- # Don't do bookkeeping if you don't want to
- if not os.path.isfile(options.pkl):
- raise Exception("%s does not exist\n" % options.pkl)
- experiment = loadExperimentFile(options.pkl)
- optimizer = experiment.optimizer
-
- # This has the side-effect of adding a job
- trial_index = get_trial_index(experiment, options.fold, params)
-
- experiment.set_one_fold_running(trial_index, options.fold)
- del experiment # release lock
-
- # TODO: Forward seed to run_instance, because like this it is useless
- result, duration, status = run_instance(options.fold, fn, params, cfg)
-
- # Do bookkeeping after the run_instance
- if options.pkl is not None:
- # Don't do bookkeeping if we don't want you to do
- if not os.path.isfile(options.pkl):
- raise Exception("%s does not exist\n" % paramfile)
- experiment = loadExperimentFile(options.pkl)
- optimizer = experiment.optimizer
- if status == "SAT":
- experiment.set_one_fold_complete(trial_index, options.fold, result,
- duration)
- elif status == "CRASHED":
- result = cfg.getfloat("DEFAULT", "result_on_terminate")
- experiment.set_one_fold_crashed(trial_index, options.fold, result, duration)
- else:
- # TODO: We need a global stopping mechanism
- sys.exit(1)
- del experiment #release lock
-
- print "Result for ParamILS: %s, %d, 1, %f, %d, %s for %s" % \
- (status, abs(duration), result, seed, optimizer, str(fn_path))
-
-if __name__ == "__main__":
- main()
diff --git a/run_test.py b/run_test.py
deleted file mode 100644
index d237f8b3..00000000
--- a/run_test.py
+++ /dev/null
@@ -1,105 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#!/usr/bin/env python
-
-import cPickle
-from importlib import import_module
-from optparse import OptionParser
-import os
-import re
-import subprocess
-import sys
-import time
-
-from gzip import GzipFile as gfile
-import numpy as np
-
-from config_parser.parse import parse_config
-from Experiment import Experiment
-import wrapping_util
-
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__contact__ = "automl.org"
-
-
-def loadExperimentFile():
- optimizer = os.getcwd().split("/")[-1].split("_")[0]
- experiment = Experiment(".", optimizer)
- return experiment
-
-def main():
- # Parse options and arguments
- usage = "Coming soon"
- parser = OptionParser(usage=usage)
- (options, args) = parser.parse_args()
-
- if len(args) != 1:
- raise Exception("You must specify a directory.")
-
- # Then load config.cfg and function
- cfg = parse_config("config.cfg", allow_no_value=True)
- module = cfg.get("DEFAULT", "function")
-
- os.chdir(args[0])
-
- # Load the experiment pickle and extract best configuration according to the
- # validation loss
- experiment = loadExperimentFile()
- optimizer = experiment.optimizer
- if optimizer == "smac":
- tmp_results = experiment.instance_results
- tmp_results = np.ma.masked_array(tmp_results, np.isnan(tmp_results))
- results = np.mean(tmp_results, axis=1)
- else:
- tmp_results = experiment.results
- tmp_results[np.isnan(tmp_results)] = cfg.get('DEFAULT', 'result_on_terminate')
- results = tmp_results
- best_config_idx = np.argmin(results)
- print "Found best config #%d with validation loss %f" % (best_config_idx, results[best_config_idx])
- params = experiment.params[best_config_idx]
- del experiment #release lock
-
- # Create a param pickle
- time_string = wrapping_util.get_time_string()
- params_filename = os.path.join(os.getcwd(), "params" + time_string)
- params_fh = open(params_filename, 'w')
- print "Pickling param dict", params_filename, params
- cPickle.dump(params, params_fh)
- params_fh.close()
-
- #Call run_instance.py
- fh = open(args[0][0:-1] + "_test_run.out", "w")
- leading_runsolver_info = cfg.get('DEFAULT', 'leading_runsolver_info')
- leading_algo_info = cfg.get('DEFAULT', 'leading_algo_info')
- leading_algo_info = "optirun"
- cmd = "%s %s python %s --fold 1 --config ../config.cfg --test %s" % (leading_runsolver_info, leading_algo_info, os.path.join(os.path.dirname(os.path.realpath(__file__)) + "/run_instance.py"), params_filename)
- process = subprocess.Popen(cmd, stdout=fh, stderr=fh,
- shell=True, executable="/bin/bash")
-
- print
- print cmd
- print "-----------------------RUNNING TEST----------------------------"
- ret = process.wait()
- fh.close()
-
- os.remove(params_filename)
- print ret
-
-if __name__ == "__main__":
- main()
- #test()
diff --git a/runsolver_wrapper.py b/runsolver_wrapper.py
deleted file mode 100644
index 1900198b..00000000
--- a/runsolver_wrapper.py
+++ /dev/null
@@ -1,266 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-#!/usr/bin/env python
-
-import cPickle
-import numpy as np
-import os
-import sys
-import time
-import subprocess
-
-from config_parser.parse import parse_config
-import Experiment
-import wrapping_util
-
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__contact__ = "automl.org"
-
-
-def load_experiment_file():
- optimizer = os.getcwd().split("/")[-1].split("_")[0]
- experiment = Experiment.Experiment(".", optimizer)
- return experiment
-
-
-def replace_nan_in_last_trial(replacement):
- exp = load_experiment_file()
- # TODO: Make a better test to find out if we are supposed to write to exp.
- if len(exp.trials) == 0 and len(exp.instance_order) == 0:
- del exp
- else:
- _id, fold = exp.instance_order[-1]
- if not np.isfinite(exp.trials[_id]['instance_results'][fold]):
- exp.trials[_id]['instance_results'][fold] = replacement
- exp.trials[_id]['instance_status'][fold] == Experiment.BROKEN_STATE
- elif exp.trials[_id]['instance_results'][fold] == replacement:
- pass
- else:
- raise Exception("Trying to replace %f with %f" %
- (exp.trials[_id]['instance_results'][fold], replacement))
- exp._save_jobs()
- del exp # This also saves the experiment
-
-
-def read_runsolver_output(runsolver_output_file):
- """
- Read the runsolver output, watch out for
- Mem limit exceeded: sending SIGTERM then SIGKILL
- Maximum CPU time exceeded: sending SIGTERM then SIGKILL
- Maximum wall clock time exceeded: sending SIGTERM then SIGKILL
- Maximum VSize exceeded: sending SIGTERM then SIGKILL
- In case one of these happened, send back the worst possible result
- as specified in the config
- """
- with open(runsolver_output_file, 'r') as f:
- runsolver_output_content = f.readlines()
- limit_exceeded = None
- error_time = 0
- for line in runsolver_output_content:
- if "Maximum CPU time exceeded" in line:
- limit_exceeded = "CPU time exceeded"
- if "Maximum wall clock time exceeded" in line:
- limit_exceeded = "Wall clock time exceeded"
- if "Maximum VSize exceeded" in line:
- limit_exceeded = "VSize exceeded"
- if "Mem limit exceeded" in line:
- limit_exceeded = "Memory exceeded"
- if "Real time (s): " in line:
- # If the solver terminated, get the wallclock time of termination
- error_time = float(line.split()[3])
- return limit_exceeded, error_time
-
-
-def read_run_instance_output(run_instance_output):
- """
- Read the run_instance output file
- """
- result_string = None
- result_array = None
- fh = open(run_instance_output, "r")
- run_instance_content = fh.readlines()
- fh.close()
- result_string = None
- for line in run_instance_content:
- pos = line.find("Result for ParamILS:")
- if pos != -1:
- result_string = line[pos:]
- result_array = result_string.split()
- break
-
- return result_array, result_string
-
-
-def get_config():
- cfg_filename = "config.cfg"
- if not os.path.isfile(cfg_filename):
- cfg_filename = "../config.cfg"
- if not os.path.isfile(cfg_filename):
- raise Exception('Could not find config.cfg in ./ or .// from %s\n'
- % os.getcwd())
-
- cfg = parse_config(cfg_filename, allow_no_value=True)
- return cfg, cfg_filename
-
-
-def main():
- # Parse options and arguments
- usage = "This script pickles the params and runs the runsolver with " +\
- "run_instance and extract the output for the optimizer \n" + \
- "The output is printed im a SMACish way: \n\n" + \
- "'Result for ParamILS: , , , " + \
- ", , ' \n\n" + \
- "Usage: runsolver_wrapper " + \
- " " + \
- " \n" + \
- " might be the optimizer name if not" + \
- " called by smac\n"
- if len(sys.argv) < 7:
- sys.stdout.write(usage)
- exit(1)
-
- optimizer = os.getcwd().split("/")[-1].split("_")[0]
-
- # This has to be done here for SMAC, since smac does not call cv.py
- if optimizer == 'smac':
- cv_starttime = time.time()
- experiment = load_experiment_file()
- experiment.start_cv(cv_starttime)
- del experiment
-
- # Again we need to find the config.cfg
- cfg, cfg_filename = get_config()
-
- # Ignore smac cutofftime
- time_limit = cfg.getint('DEFAULT', 'runsolver_time_limit')
- memory_limit = cfg.getint('DEFAULT', 'memory_limit')
- cpu_limit = cfg.getint('DEFAULT', 'cpu_limit')
-
- # Then get some information for run_instance
- fold = int(sys.argv[1])
- seed = int(sys.argv[5])
-
- # Now build param dict
- param_list = sys.argv[6:]
- params = dict()
-
- for idx, i in enumerate(param_list[0::2]):
- params[param_list[idx*2][1:]] = (param_list[idx*2+1].strip("'"))
- # TODO: remove overhead of pickling the param dict
- time_string = wrapping_util.get_time_string()
- params_filename = os.path.join(os.getcwd(), "params" + time_string)
- params_fh = open(params_filename, 'w')
- print 'Pickling param dict', params_filename, params
- cPickle.dump(params, params_fh)
- params_fh.close()
-
- # Timestamp and use-pty are options are used so that output of the "solver"
- # is flushed to the solver output file directly
- run_instance_output = os.path.join(os.getcwd(), time_string + "_run_instance.out")
- sys.stdout.write("Using optimizer: " + str(optimizer) + "\n")
-
- python_cmd = cfg.get("DEFAULT", "leading_algo_info") + " python " + \
- os.path.join(os.path.dirname(os.path.realpath(__file__)),
- "run_instance.py") + \
- " --fold %d --config %s --seed %d --pkl %s" % \
- (fold, cfg_filename, seed, optimizer + ".pkl")
- # Do not write the actual task in quotes because runsolver will not work
- # then
- delay = 0
- cmd = cfg.get("DEFAULT", "leading_runsolver_info") + \
- " runsolver -o %s --timestamp --use-pty -W %d -C %d -M %d -d %d %s %s" \
- % (run_instance_output, time_limit, cpu_limit, memory_limit, delay,
- python_cmd, params_filename)
-
- runsolver_output_file = os.path.join(os.getcwd(),
- time_string + "_runsolver.out")
- fh = open(runsolver_output_file, "w")
- process = subprocess.Popen(cmd, stdout=fh,
- stderr=fh, shell=True, executable="/bin/bash")
-
- print
- print cmd
- print "-----------------------RUNNING RUNSOLVER----------------------------"
- process.wait()
- fh.close()
-
- limit_exceeded, error_time = read_runsolver_output(runsolver_output_file)
- # , , , , ,
- error_string = "Result for ParamILS: %s, %d, 0, %f, %d, %s"
-
- result_array, result_string = read_run_instance_output(run_instance_output)
-
- # Write the SMACish output to the command line and remove temporary files
- # Additionaly, we also have to replace the NaN-value in the result file by
- # the worst possible value
- if result_array is not None and result_array[3].strip(",") not in ("SAT", "CRASHED"):
- raise Exception("Unknown return status %s" % result_array[3].strip(","))
-
- if (limit_exceeded is None and result_string is None) or\
- (result_string is not None and "CRASHED" in result_string and not limit_exceeded):
- replace_nan_in_last_trial(cfg.getfloat("DEFAULT", "result_on_terminate"))
- return_string = error_string % ("SAT", error_time,
- cfg.getfloat('DEFAULT', "result_on_terminate"),
- seed,
- "Please have a look at " + run_instance_output)
- print return_string
- os.remove(runsolver_output_file)
-
- elif limit_exceeded is None:
- result_float = float(result_array[6].strip(","))
- if not np.isfinite(result_float):
- replace_nan_in_last_trial()
- experiment = load_experiment_file()
- last_experiment = experiment.instance_order[-1]
- assert last_experiment is not None
- experiment.instance_results[last_experiment[0]][last_experiment[1]]\
- = cfg.getfloat('DEFAULT', "result_on_terminate")
- del experiment # This also saves the experiment
-
- result_array[6] = (cfg.getfloat("DEFAULT", "result_on_terminate")) + ","
- result_string = " ".join(result_array)
- else:
- # Remove the run_instance_output only if there is a valid result
- os.remove(run_instance_output)
- os.remove(runsolver_output_file)
- return_string = result_string
- print result_string
-
- else:
- return_string = error_string % ("SAT", error_time, cfg.getfloat
- ('DEFAULT', "result_on_terminate"), seed,
- limit_exceeded)
- print return_string
- # It is useful to have the run_instance_output for debugging
- #os.remove(run_instance_output)
-
- os.remove(params_filename)
-
- # Remove param pkl and runsolver files
- #os.remove(run_instance_output)
- #os.remove(runsolver_output_file)
- #os.remove(params_filename)
- if optimizer == 'smac':
- experiment = load_experiment_file()
- experiment.end_cv(time.time())
- del experiment
- return return_string
-
-if __name__ == "__main__":
- main()
diff --git a/optimizers/spearmint_april2013_mod/__init__.py b/scripts/HPOlib-convert
old mode 100755
new mode 100644
similarity index 65%
rename from optimizers/spearmint_april2013_mod/__init__.py
rename to scripts/HPOlib-convert
index 4b9bc013..94f2a43c
--- a/optimizers/spearmint_april2013_mod/__init__.py
+++ b/scripts/HPOlib-convert
@@ -1,21 +1,26 @@
+#!/usr/bin/env python
+
##
-# Copyright (C) 2012 Jasper Snoek, Hugo Larochelle and Ryan P. Adams
-#
-# This code is written for research and educational purposes only to
-# supplement the paper entitled
-# "Practical Bayesian Optimization of Machine Learning Algorithms"
-# by Snoek, Larochelle and Adams
-# Advances in Neural Information Processing Systems, 2012
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
-#
+#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
-#
+#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+from HPOlib.format_converter import convert
+
+convert.main()
\ No newline at end of file
diff --git a/scripts/HPOlib-plot b/scripts/HPOlib-plot
new file mode 100644
index 00000000..3b1aecba
--- /dev/null
+++ b/scripts/HPOlib-plot
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+from HPOlib.Plotting import doAllPlots
+
+doAllPlots.main()
\ No newline at end of file
diff --git a/scripts/HPOlib-run b/scripts/HPOlib-run
new file mode 100755
index 00000000..eb5485b4
--- /dev/null
+++ b/scripts/HPOlib-run
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+from HPOlib import wrapping
+
+wrapping.main()
\ No newline at end of file
diff --git a/setup.py b/setup.py
new file mode 100644
index 00000000..13a9f8e2
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,280 @@
+import hashlib
+from setuptools import setup
+from setuptools.command.install import install
+import shutil
+import urllib
+import os
+import subprocess
+import sys
+import tarfile
+
+import HPOlib
+
+here = os.path.abspath(os.path.dirname(__file__))
+desc = 'A software that can be used to evaluate either black boxes or hyperparameter optimization algorithms'
+keywords = 'hyperparameter optimization empirical evaluation black box'
+
+package_dir = {'HPOlib': 'HPOlib',
+ 'HPOlib.config_parser': 'HPOlib/config_parser',
+ 'HPOlib.Plotting': 'HPOlib/Plotting',
+ 'HPOlib.format_converter': 'HPOlib/format_converter'}
+package_data = {'HPOlib.config_parser': ['*.cfg']}
+
+data_files = []
+scripts = ['scripts/HPOlib-run', 'scripts/HPOlib-plot', 'runsolver/src/runsolver', 'scripts/HPOlib-convert']
+
+
+def read(fname):
+ return open(os.path.join(os.path.dirname(__file__), fname)).read()
+
+
+def get_find_packages():
+ packages = ['HPOlib',
+ 'HPOlib.config_parser',
+ 'HPOlib.Plotting',
+ 'HPOlib.format_converter']
+ return packages
+
+
+def download_source(download_url, md5, save_as):
+ try:
+ urllib.urlretrieve(download_url, filename=save_as)
+ except Exception, e:
+ sys.stdout.write("Error downloading %s: %s\n" % (download_url, e))
+ return False
+ md5_new = hashlib.md5(open(save_as).read()).hexdigest()
+ if md5_new != md5:
+ sys.stdout.write("md5 checksum has changed: %s to %s\n" % (md5, md5_new))
+ return False
+ return True
+
+
+def extract_source(fn_name, extract_as):
+ if tarfile.is_tarfile(fn_name):
+ try:
+ tfile = tarfile.open(fn_name)
+ tfile.extractall(extract_as)
+ except Exception, e:
+ sys.stdout.write("Error occurred during extraction: %s\n" % e)
+ return False
+ return True
+ return False
+
+
+def copy_folder(new_dir, old_dir):
+ try:
+ shutil.copytree(old_dir, new_dir, symlinks=True,
+ ignore=shutil.ignore_patterns('*.pyc', '*_src*'))
+ except shutil.Error as e:
+ sys.stderr.write("[shutil.ERROR] Could not copy folder from %s to %s\n" % (old_dir, new_dir))
+ sys.stderr.write("%s\n" % e.message)
+ return False
+ except Exception, e:
+ sys.stderr.write("[ERROR] Could not copy folder from %s to %s\n" % (old_dir, new_dir))
+ sys.stderr.write("%s\n" % e.message)
+ return False
+ return True
+
+
+def make_dir(new_dir):
+ try:
+ if not os.path.exists(new_dir):
+ os.mkdir(new_dir)
+ else:
+ sys.stdout.write("[INFO] %s is already a directory, we use it for our installation\n" %
+ new_dir)
+ except Exception, e:
+ sys.stdout.write("[ERROR] Something (%s) went wrong, please create %s and try again\n" %
+ (e.message, new_dir))
+
+
+class AdditionalInstall(install):
+ """Is supposed to install the runsolver, download optimizers,
+ and copy benchmarks"""
+
+ def _check_for_runsolver(self):
+ name = 'runsolver'
+ flags = os.X_OK
+ path = os.environ.get('PATH', None)
+ if path is None:
+ return False
+ for p in os.environ.get('PATH', '').split(os.pathsep):
+ p = os.path.join(p, name)
+ if os.access(p, flags) and not os.path.isdir(p):
+ if not os.path.isdir(os.path.join(os.getcwd(), 'runsolver/src/')):
+ os.mkdir(os.path.join(os.getcwd(), 'runsolver/src/'))
+ sys.stdout.write("Copy runsolver from %s to runsolver/src/runsolver\n" % p)
+ shutil.copy(p, os.path.join(os.getcwd(), 'runsolver/src/runsolver'))
+ return p
+ return False
+
+ def _build(self, build_dir):
+ sys.stdout.write("Building runsolver\n")
+ cur_pwd = os.getcwd()
+ os.chdir(build_dir)
+ try:
+ subprocess.check_call("make")
+ except subprocess.CalledProcessError, e:
+ sys.stdout.write("Error while building runsolver: %s\n" % e)
+ os.chdir(cur_pwd)
+ return False
+ os.chdir(cur_pwd)
+ return True
+
+ def _copy_and_download_optimizer(self, optimizer_dir, optimizer_name, optimizer_tar_name,
+ url, md5):
+ load, extract = (False, False)
+
+ load = download_source(download_url=url, md5=md5,
+ save_as=os.path.join(optimizer_dir, optimizer_tar_name))
+
+ if load:
+ extract = extract_source(os.path.join(optimizer_dir, optimizer_tar_name),
+ extract_as=os.path.join(optimizer_dir, optimizer_name))
+ os.remove(os.path.join(optimizer_dir, optimizer_tar_name))
+
+ if load and extract:
+ return True
+ else:
+ return False
+
+ def run(self):
+ # RUNSOLVER STUFF
+ here_we_are = os.getcwd()
+ runsolver_tar_name = "runsolver-3.3.4.tar.bz2"
+ runsolver_name = "runsolver-3.3.4"
+ if sys.version_info < (2, 7, 0) or sys.version_info >= (2, 8, 0):
+ sys.stderr.write("HPOlib requires Python 2.7.0\n")
+ sys.exit(-1)
+
+ downloaded, extracted, built = (False, False, False)
+ runsolver_needs_to_be_installed = False
+
+ runsolver_path = self._check_for_runsolver()
+ if not runsolver_path:
+ sys.stdout.write("[INFO] 'runsolver' not found, so we try to install it\n")
+ runsolver_needs_to_be_installed = True
+
+ if runsolver_needs_to_be_installed:
+ sys.stdout.write("Downloading runsolver from %s%s, saving to %s/%s\n" %
+ ('http://www.cril.univ-artois.fr/~roussel/runsolver/',
+ runsolver_tar_name, os.getcwd(), runsolver_tar_name))
+ downloaded = download_source(download_url='http://www.cril.univ-artois.fr/~roussel/runsolver/%s' %
+ runsolver_tar_name,
+ md5="5a9511266489c87f4a276b9e54ea4082",
+ save_as=os.path.join(here_we_are, runsolver_tar_name))
+
+ if runsolver_needs_to_be_installed and downloaded:
+ sys.stdout.write("Extracting runsolver to %s\n" % os.path.join(here_we_are, runsolver_name))
+ extracted = extract_source(fn_name=os.path.join(here_we_are, runsolver_tar_name), extract_as=here_we_are)
+
+ if runsolver_needs_to_be_installed and extracted:
+ sys.stdout.write("Building runsolver\n")
+ sys.stdout.write("Replace a line, which seems to cause trouble on 32 bit systems'\n")
+ call = "sed -i 's|gnu/asm/unistd_$(WSIZE).h|gnu/asm/unistd_$(WSIZE).h /usr/include/i386-linux-gnu/asm/unistd_$(WSIZE).h|' runsolver/src/Makefile"
+ try:
+ subprocess.check_call(call, shell=True)
+ except subprocess.CalledProcessError, e:
+ sys.stdout.write("Replacing did not work: %s\n" % e)
+ md5_new = hashlib.md5(open("runsolver/src/Makefile").read()).hexdigest()
+ runsolver_md5 = "4870722e47a6f74d5167376d371f1730"
+ if md5_new != runsolver_md5:
+ sys.stdout.write("md5 checksum has changed: %s to %s, not compiling runsolver\n" % (runsolver_md5, md5_new))
+ else:
+ built = self._build(build_dir=os.path.join(here_we_are, "runsolver", "src"))
+
+ # COPY/DOWNLOAD OPTIMIZER TO ROOT FOLDER
+ optimizer_dir = os.path.join(here_we_are, "optimizers")
+
+ tpe, smac, spearmint = (False, False, False)
+ tpe = self._copy_and_download_optimizer(optimizer_dir=optimizer_dir,
+ optimizer_name='tpe',
+ optimizer_tar_name="hyperopt_august2013_mod_src.tar.gz",
+ url="http://www.automl.org/hyperopt_august2013_mod_src.tar.gz",
+ md5='15ce1adf9d32bb7f71dcfb9a847c55c7')
+ # If one optimizer fails, others are likely to fail as well
+ if tpe:
+ smac = self._copy_and_download_optimizer(optimizer_dir=optimizer_dir,
+ optimizer_name='smac',
+ optimizer_tar_name="smac_2_06_01-dev_src.tar.gz",
+ url="http://www.automl.org/smac_2_06_01-dev_src.tar.gz",
+ md5='30ab1d09696de47efac77ed163772c0a')
+ spearmint = self._copy_and_download_optimizer(optimizer_dir=optimizer_dir,
+ optimizer_name='spearmint',
+ optimizer_tar_name="spearmint_april2013_mod_src.tar.gz",
+ url="http://www.automl.org/spearmint_april2013" +
+ "_mod_src.tar.gz",
+ md5='340fc0da97a30454d633ce9781b05369')
+
+ # Do whatever you want to do
+ # TODO: Normally one wants to call run(self), but this runs distutils and ignores install_requirements for unknown reasons
+ # if anyone knows a better way, feel free to change
+ install.do_egg_install(self)
+
+ # Give detailed output to user
+ if not tpe or not smac or not spearmint:
+ sys.stderr.write("[ERROR] Something went wrong while copying and downloading optimizers." +
+ "Please do the following to be ready to start optimizing:\n\n" +
+ "cd optimizers\n" +
+ "wget http://www.automl.org/hyperopt_august2013_mod_src.tar.gz \n" +
+ "wget http://www.automl.org/smac_2_06_01-dev_src.tar.gz \n" +
+ "wget http://www.automl.org/spearmint_april2013_mod_src.tar.gz \n" +
+ "tar -xf hyperopt_august2013_mod_src.tar.gz \n" +
+ "mv hyperopt_august2013_mod_src tpe/ \n" +
+ "tar -xf smac_2_06_01-dev_src.tar.gz \n" + "mv smac_2_06_01-dev_src.tar.gz smac/ \n" +
+ "tar -xf spearmint_april2013_mod_src.tar.gz \n" +
+ "mv spearmint_april2013_mod_src spearmint/ \n\n" +
+ "Thank You!\n")
+ if runsolver_needs_to_be_installed and not built:
+ sys.stdout.write("[ERROR] Please install runsolver on your own! You can download it from:\n%s%s\n" % \
+ ('http://www.cril.univ-artois.fr/~roussel/runsolver/%s', runsolver_tar_name))
+ sys.stdout.write("[ERROR] Be sure 'runsolver' can be found during runtime in your $PATH variable!\n")
+ if not runsolver_needs_to_be_installed and not built:
+ sys.stdout.write("[INFO] 'runsolver' has been found on this system and was copied from: %s\n" %
+ runsolver_path)
+ if built:
+ sys.stdout.write("'runsolver' has been downloaded and installed\n")
+
+setup(
+ name='HPOlib',
+ version=HPOlib.__version__,
+ url='https://github.com/automl/HPOlib',
+ license='LGPLv3',
+ platforms=['Linux'],
+ author=HPOlib.__authors__,
+ test_suite="tests.testsuite.suite",
+ # setup_requires=['numpy'],
+ # We require scipy, but this does not automatically install numpy,
+ # so the user needs to make sure numpy is already installed
+ install_requires=['argparse','numpy',
+ 'matplotlib',
+ 'networkx',
+ 'protobuf',
+ 'scipy>=0.13.2',
+ 'pymongo'
+ ],
+ author_email='eggenspk@informatik.uni-freiburg.de',
+ description=desc,
+ long_description=read("README.md"),
+ keywords=keywords,
+ packages=get_find_packages(),
+ package_dir=package_dir,
+ package_data=package_data,
+ data_files=data_files,
+ scripts=scripts,
+ cmdclass={'install': AdditionalInstall},
+ classifiers=[
+ 'Programming Language :: Python :: 2.7',
+ 'Development Status :: 3 - Alpha',
+ 'Natural Language :: English',
+ 'Environment :: Console',
+ 'Intended Audience :: Developers',
+ 'Intended Audience :: Education',
+ 'Intended Audience :: Science/Research',
+ 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)',
+ 'Operating System :: POSIX :: Linux',
+ 'Topic :: Scientific/Engineering :: Artificial Intelligence',
+ 'Topic :: Scientific/Engineering',
+ 'Topic :: Software Development',
+ ]
+)
diff --git a/optimizers/hyperopt_august2013_mod/__init__.py b/tests/__init__.py
similarity index 100%
rename from optimizers/hyperopt_august2013_mod/__init__.py
rename to tests/__init__.py
diff --git a/tests/testsuite.py b/tests/testsuite.py
new file mode 100644
index 00000000..f69f973b
--- /dev/null
+++ b/tests/testsuite.py
@@ -0,0 +1,30 @@
+import unittest
+
+import unittests.test_benchmark_util as test_benchmark_util
+# import unittests.test_cv as test_cv
+import unittests.test_data_utils as test_data_utils
+import unittests.test_experiment as test_experiment
+#import unittests.test_gridsearch as test_gridsearch
+import unittests.test_runsolver_wrapper as test_runsolver_wrapper
+import unittests.test_wrapping as test_wrapping
+import unittests.test_wrapping_util as test_wrapping_util
+
+
+def suite():
+ _suite = unittest.TestSuite()
+ _suite.addTest(unittest.makeSuite(test_benchmark_util.BenchmarkUtilTest))
+ # _suite.addTest(unittest.makeSuite(test_cv.CVTest))
+ _suite.addTest(unittest.makeSuite(test_data_utils.DataUtilTest))
+ _suite.addTest(unittest.makeSuite(test_experiment.ExperimentTest))
+ #_suite.addTest(unittest.makeSuite(test_gridsearch.GridSearchTest))
+ _suite.addTest(unittest.makeSuite(test_runsolver_wrapper.RunsolverWrapperTest))
+ _suite.addTest(unittest.makeSuite(test_wrapping.WrappingTest))
+ _suite.addTest(unittest.makeSuite(test_wrapping_util.WrappingTestUtil))
+ return _suite
+
+
+if __name__ == "__main__":
+ runner = unittest.TextTestRunner(verbosity=2)
+ test_suite = suite()
+ runner.run(suite())
+
diff --git a/tests/unittests/__init__.py b/tests/unittests/__init__.py
new file mode 100644
index 00000000..8f0ce6cb
--- /dev/null
+++ b/tests/unittests/__init__.py
@@ -0,0 +1 @@
+__author__ = 'feurerm'
diff --git a/optimizers/smac_2_06_01-dev/dummy.py b/tests/unittests/branin_test/__init__.py
similarity index 100%
rename from optimizers/smac_2_06_01-dev/dummy.py
rename to tests/unittests/branin_test/__init__.py
diff --git a/tests/unittests/branin_test/branin.py b/tests/unittests/branin_test/branin.py
new file mode 100644
index 00000000..c1210a90
--- /dev/null
+++ b/tests/unittests/branin_test/branin.py
@@ -0,0 +1,51 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import sys
+from numpy import NaN, pi, cos, ndarray
+
+def branin(params, **kwargs):
+# branin function
+# The number of variables n = 2.
+# constraints:
+# -5 <= x <= 10, 0 <= y <= 15
+# three global optima: (-pi, 12.275), (pi, 2.275), (9.42478, 2.475), where
+# branin = 0.397887
+ if "x" not in params or "y" not in params:
+ raise ValueError("No params found ['x', 'y']\n")
+ x = float(params["x"])
+ y = float(params["y"])
+ if type(x) == ndarray or type(y) == ndarray:
+ x = x[0]
+ y = y[0]
+
+ if -5 > x or x > 10:
+ raise ValueError("X value not in between -5 and 10")
+ if 0 > y or y > 15:
+ raise ValueError("Y value not in between 0 and 15")
+
+ result = (y-(5.1/(4*pi**2))*x**2+5*x/pi-6)**2
+ result += 10*(1-1/(8*pi))*cos(x)+10
+ return result
+
+
+def main(params, **kwargs):
+ print 'Params: ', params,
+ y = branin(params, **kwargs)
+ print 'Result: ', y
+ return y
diff --git a/tests/unittests/branin_test/config.cfg b/tests/unittests/branin_test/config.cfg
new file mode 100644
index 00000000..13017468
--- /dev/null
+++ b/tests/unittests/branin_test/config.cfg
@@ -0,0 +1,8 @@
+[SMAC]
+p = smac/params.txt
+
+[DEFAULT]
+function = ./branin.py
+number_of_jobs = 2
+result_on_terminate = 1000
+number_cv_folds = 5
diff --git a/tests/unittests/branin_test/params123 b/tests/unittests/branin_test/params123
new file mode 100644
index 00000000..c0076ed9
--- /dev/null
+++ b/tests/unittests/branin_test/params123
@@ -0,0 +1,6 @@
+(dp1
+S'y'
+S'5'
+sS'x'
+S'5'
+s.
\ No newline at end of file
diff --git a/tests/unittests/branin_test/random/space.py b/tests/unittests/branin_test/random/space.py
new file mode 100644
index 00000000..cccaea4d
--- /dev/null
+++ b/tests/unittests/branin_test/random/space.py
@@ -0,0 +1,4 @@
+from hyperopt import hp
+
+space = {'x': hp.uniform('x', -5, 10),
+ 'y': hp.uniform('y', 0, 15) }
\ No newline at end of file
diff --git a/tests/unittests/branin_test/smac/params.txt b/tests/unittests/branin_test/smac/params.txt
new file mode 100644
index 00000000..72b78373
--- /dev/null
+++ b/tests/unittests/branin_test/smac/params.txt
@@ -0,0 +1,2 @@
+x [-5, 10] [2.5]
+y [0, 15] [7.5]
diff --git a/tests/unittests/branin_test/spearmint/__init__.py b/tests/unittests/branin_test/spearmint/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/benchmarks/branin/spearmint/config.pb b/tests/unittests/branin_test/spearmint/config.pb
similarity index 100%
rename from benchmarks/branin/spearmint/config.pb
rename to tests/unittests/branin_test/spearmint/config.pb
diff --git a/tests/unittests/branin_test/tpe/space.py b/tests/unittests/branin_test/tpe/space.py
new file mode 100644
index 00000000..cccaea4d
--- /dev/null
+++ b/tests/unittests/branin_test/tpe/space.py
@@ -0,0 +1,4 @@
+from hyperopt import hp
+
+space = {'x': hp.uniform('x', -5, 10),
+ 'y': hp.uniform('y', 0, 15) }
\ No newline at end of file
diff --git a/tests/unittests/dummy_config.cfg b/tests/unittests/dummy_config.cfg
new file mode 100644
index 00000000..196ef165
--- /dev/null
+++ b/tests/unittests/dummy_config.cfg
@@ -0,0 +1,7 @@
+[HPOLIB]
+number_of_jobs = 1
+result_on_terminate = 1
+function = 1
+
+[GRIDSEARCH]
+params = params.pcs
\ No newline at end of file
diff --git a/tests/unittests/run_instance_no_result.txt b/tests/unittests/run_instance_no_result.txt
new file mode 100644
index 00000000..6134ff51
--- /dev/null
+++ b/tests/unittests/run_instance_no_result.txt
@@ -0,0 +1,4 @@
+0.05/0.17 Traceback (most recent call last):
+0.05/0.17 File "/Software/HPOlib/HPOlib/benchmarks/branin/branin.py", line 21, in
+0.05/0.17 import benchmark_util
+0.05/0.17 ImportError: No module named benchmark_util
\ No newline at end of file
diff --git a/tests/unittests/run_instance_result.txt b/tests/unittests/run_instance_result.txt
new file mode 100644
index 00000000..b36d9a5c
--- /dev/null
+++ b/tests/unittests/run_instance_result.txt
@@ -0,0 +1 @@
+00.01/00.12 Result for ParamILS: SAT, 0.35, 1, 0.5, -1, Random file
\ No newline at end of file
diff --git a/tests/unittests/runsolver_positive.txt b/tests/unittests/runsolver_positive.txt
new file mode 100644
index 00000000..0f869da8
--- /dev/null
+++ b/tests/unittests/runsolver_positive.txt
@@ -0,0 +1,69 @@
+runsolver Copyright (C) 2010-2013 Olivier ROUSSEL
+
+This is runsolver version 3.3.2 (svn: 1389)
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+command line: runsolver -o /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/branin/smac_2_06_01-dev_1_2014-2-4--13-34-14-689426/2014-2-4--13-37-6-214300_run_instance.out --timestamp --use-pty -W 3600 -C 14400 -M 2000 -d 0 python /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/branin/branin.py --fold 0 --folds 1 --params -x2.5 -y7.5
+
+running on 4 cores: 0-3
+
+Enforcing wall clock limit (soft limit, will send SIGTERM then SIGKILL): 3600 seconds
+Enforcing CPUTime limit (soft limit, will send SIGTERM then SIGKILL): 14400 seconds
+Enforcing CPUTime limit (hard limit, will send SIGXCPU): 14430 seconds
+Enforcing VSIZE limit (soft limit, will send SIGTERM then SIGKILL): 2048000 KiB
+Enforcing VSIZE limit (hard limit, stack expansion will fail with SIGSEGV, brk() and mmap() will return ENOMEM): 2099200 KiB
+Using a pseudo terminal to collect output from the solver
+Current StackSize limit: 8192 KiB
+
+
+[startup+0 s]
+/proc/loadavg: 0.93 0.47 0.34 3/498 26603
+/proc/meminfo: memFree=5018208/8174740 swapFree=0/0
+[pid=26603] ppid=26601 vsize=7764 CPUtime=0 cores=0-3
+/proc/26603/stat : 26603 (python) R 26601 26603 19549 34817 26580 4202496 591 0 0 0 0 0 0 0 20 0 1 0 60173368 7950336 402 4294967295 134512640 136958856 3216281168 3216280624 135835648 0 0 16781312 0 0 0 0 17 1 0 0 0 0 0 136965844 137310660 161521664 3216283729 3216283848 3216283848 3216285641 0
+/proc/26603/statm: 1941 402 305 598 0 159 0
+
+Solver just ended. Dumping a history of the last processes samples
+
+[startup+0.200325 s]
+/proc/loadavg: 0.93 0.47 0.34 3/498 26603
+/proc/meminfo: memFree=5018208/8174740 swapFree=0/0
+[pid=26603] ppid=26601 vsize=35076 CPUtime=0.13 cores=0-3
+/proc/26603/stat : 26603 (python) R 26601 26603 19549 34817 26580 4202496 2864 0 0 0 9 4 0 0 20 0 1 0 60173368 35917824 2147 4294967295 134512640 136958856 3216281168 3216245016 3078460452 0 0 16781312 2 0 0 0 17 0 0 0 0 0 0 136965844 137310660 161521664 3216283729 3216283848 3216283848 3216285641 0
+/proc/26603/statm: 8513 2172 909 598 0 1461 0
+Current children cumulated CPU time (s) 0.13
+Current children cumulated vsize (KiB) 35076
+
+Child status: 1
+Real time (s): 0.259524
+CPU time (s): 0.188011
+CPU user time (s): 0.140008
+CPU system time (s): 0.048003
+CPU usage (%): 72.4446
+Max. virtual memory (cumulated for all children) (KiB): 35076
+
+getrusage(RUSAGE_CHILDREN,...) data:
+user time used= 0.140008
+system time used= 0.048003
+maximum resident set size= 11280
+integral shared memory size= 0
+integral unshared data size= 0
+integral unshared stack size= 0
+page reclaims= 3769
+page faults= 0
+swaps= 0
+block input operations= 0
+block output operations= 8
+messages sent= 0
+messages received= 0
+signals received= 0
+voluntary context switches= 206
+involuntary context switches= 31
+
+runsolver used 0.012 second user time and 0.012 second system time
+
+The end
\ No newline at end of file
diff --git a/tests/unittests/runsolver_positive_with_warning.txt b/tests/unittests/runsolver_positive_with_warning.txt
new file mode 100644
index 00000000..db1db64e
--- /dev/null
+++ b/tests/unittests/runsolver_positive_with_warning.txt
@@ -0,0 +1,147 @@
+runsolver Copyright (C) 2010-2013 Olivier ROUSSEL
+
+This is runsolver version 3.3.2 (svn: 1389)
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+command line: runsolver -o /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/libsvm/gridsearch_1_2014-2-11--14-16-48-480307/2014-2-11--14-18-27-126358_run_instance.out --timestamp --use-pty -W 3600 -C 14400 -M 2000 -d 0 python /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/libsvm/libsvm.py --fold 8 --folds 10 --params -C -5 -gamma -12
+
+running on 4 cores: 0-3
+
+Enforcing wall clock limit (soft limit, will send SIGTERM then SIGKILL): 3600 seconds
+Enforcing CPUTime limit (soft limit, will send SIGTERM then SIGKILL): 14400 seconds
+Enforcing CPUTime limit (hard limit, will send SIGXCPU): 14430 seconds
+Enforcing VSIZE limit (soft limit, will send SIGTERM then SIGKILL): 2048000 KiB
+Enforcing VSIZE limit (hard limit, stack expansion will fail with SIGSEGV, brk() and mmap() will return ENOMEM): 2099200 KiB
+Using a pseudo terminal to collect output from the solver
+Current StackSize limit: 8192 KiB
+
+
+[startup+0 s]
+/proc/loadavg: 1.16 0.82 0.65 5/544 30014
+/proc/meminfo: memFree=3713800/8174740 swapFree=12/12
+[pid=30014] ppid=30012 vsize=7908 CPUtime=0 cores=0-3
+/proc/30014/stat : 30014 (python) R 30012 30014 26453 34818 28623 4202496 775 0 0 0 0 0 0 0 20 0 1 0 41695493 8097792 617 4294967295 134512640 136958856 3218443616 3218407092 3074162534 0 0 16781312 2 0 0 0 17 0 0 0 0 0 0 136965844 137310660 170483712 3218446272 3218446397 3218446397 3218448329 0
+/proc/30014/statm: 1977 617 421 598 0 195 0
+
+[startup+0.062109 s]
+/proc/loadavg: 1.16 0.82 0.65 5/544 30014
+/proc/meminfo: memFree=3713800/8174740 swapFree=12/12
+[pid=30014] ppid=30012 vsize=12236 CPUtime=0.03 cores=0-3
+/proc/30014/stat : 30014 (python) R 30012 30014 26453 34818 28623 4202496 1210 0 0 0 3 0 0 0 20 0 1 0 41695493 12529664 990 4294967295 134512640 136958856 3218443616 3218442812 135527088 0 0 16781312 2 0 0 0 17 0 0 0 0 0 0 136965844 137310660 170483712 3218446272 3218446397 3218446397 3218448329 0
+/proc/30014/statm: 3059 990 572 598 0 395 0
+Current children cumulated CPU time (s) 0.03
+Current children cumulated vsize (KiB) 12236
+
+[startup+0.10047 s]
+/proc/loadavg: 1.16 0.82 0.65 5/544 30014
+/proc/meminfo: memFree=3713800/8174740 swapFree=12/12
+[pid=30014] ppid=30012 vsize=14928 CPUtime=0.06 cores=0-3
+/proc/30014/stat : 30014 (python) R 30012 30014 26453 34818 28623 4202496 1599 0 0 0 5 1 0 0 20 0 1 0 41695493 15286272 1256 4294967295 134512640 136958856 3218443616 3218346768 3073911061 0 0 16781312 2 0 0 0 17 1 0 0 0 0 0 136965844 137310660 170483712 3218446272 3218446397 3218446397 3218448329 0
+/proc/30014/statm: 3732 1256 664 598 0 604 0
+Current children cumulated CPU time (s) 0.06
+Current children cumulated vsize (KiB) 14928
+
+[startup+0.300223 s]
+/proc/loadavg: 1.16 0.82 0.65 5/544 30014
+/proc/meminfo: memFree=3713800/8174740 swapFree=12/12
+[pid=30014] ppid=30012 vsize=36928 CPUtime=0.19 cores=0-3
+/proc/30014/stat : 30014 (python) D 30012 30014 26453 34818 28623 1077944320 3850 0 0 0 16 3 0 0 20 0 1 0 41695493 37814272 2845 4294967295 134512640 136958856 3218443616 3218380776 3077809188 0 0 16781312 2 4187962909 0 0 17 0 0 0 0 0 0 136965844 137310660 170483712 3218446272 3218446397 3218446397 3218448329 0
+/proc/30014/statm: 9232 2845 1034 598 0 2001 0
+Current children cumulated CPU time (s) 0.19
+Current children cumulated vsize (KiB) 36928
+
+[startup+0.700156 s]
+/proc/loadavg: 1.16 0.82 0.65 5/544 30014
+/proc/meminfo: memFree=3713800/8174740 swapFree=12/12
+[pid=30014] ppid=30012 vsize=58548 CPUtime=0.37 cores=0-3
+/proc/30014/stat : 30014 (python) D 30012 30014 26453 34818 28623 1077944320 7416 0 0 0 29 8 0 0 20 0 1 0 41695493 59953152 5552 4294967295 134512640 136958856 3218443616 3218376472 3077809188 0 0 16781312 2 4187962909 0 0 17 0 0 0 0 0 0 136965844 137310660 170483712 3218446272 3218446397 3218446397 3218448329 0
+/proc/30014/statm: 14637 5552 1786 598 0 4011 0
+Current children cumulated CPU time (s) 0.37
+Current children cumulated vsize (KiB) 58548
+
+############
+# WARNING:
+# current cumulated CPU time (0 s) is less than in the last sample (0.38 s)
+# The time of a child was probably not reported to its father.
+# (see the two samples below)
+# Adding the difference (0.38 s) to the 'lost time'.
+
+[startup+0.900147 s]
+/proc/loadavg: 1.16 0.82 0.65 5/544 30014
+/proc/meminfo: memFree=3713800/8174740 swapFree=12/12
+[pid=30014] ppid=30012 vsize=58944 CPUtime=0.38 cores=0-3
+/proc/30014/stat : 30014 (python) D 30012 30014 26453 34818 28623 1077944320 7538 0 0 0 30 8 0 0 20 0 1 0 41695493 60358656 5615 4294967295 134512640 136958856 3218443616 3218411784 3077809188 0 0 16781312 2 4187962909 0 0 17 0 0 0 0 0 0 136965844 137310660 170483712 3218446272 3218446397 3218446397 3218448329 0
+/proc/30014/statm: 14736 5615 1792 598 0 4091 0
+Current children cumulated CPU time (s) 0.38
+Current children cumulated vsize (KiB) 58944
+
+[startup+1.00023 s]
+/proc/loadavg: 1.16 0.82 0.65 3/546 30020
+/proc/meminfo: memFree=3725896/8174740 swapFree=12/12
+[pid=30014] ppid=-1 vsize=0 CPUtime=0 cores=
+/proc/30014/stat : -- couldn't open stat file --/proc/30014/statm: Current children cumulated CPU time (s) 0
+Current children cumulated vsize (KiB) 0
+#
+############
+
+
+Solver just ended. Dumping a history of the last processes samples
+
+[startup+0.900147 s]
+/proc/loadavg: 1.16 0.82 0.65 5/544 30014
+/proc/meminfo: memFree=3713800/8174740 swapFree=12/12
+[pid=30014] ppid=30012 vsize=58944 CPUtime=0.38 cores=0-3
+/proc/30014/stat : 30014 (python) D 30012 30014 26453 34818 28623 1077944320 7538 0 0 0 30 8 0 0 20 0 1 0 41695493 60358656 5615 4294967295 134512640 136958856 3218443616 3218411784 3077809188 0 0 16781312 2 4187962909 0 0 17 0 0 0 0 0 0 136965844 137310660 170483712 3218446272 3218446397 3218446397 3218448329 0
+/proc/30014/statm: 14736 5615 1792 598 0 4091 0
+Current children cumulated CPU time (s) 0.38
+Current children cumulated vsize (KiB) 58944
+
+[startup+1.00023 s]
+/proc/loadavg: 1.16 0.82 0.65 3/546 30020
+/proc/meminfo: memFree=3725896/8174740 swapFree=12/12
+[pid=30014] ppid=-1 vsize=0 CPUtime=0 cores=
+/proc/30014/stat : -- couldn't open stat file --/proc/30014/statm: Current children cumulated CPU time (s) 0
+Current children cumulated vsize (KiB) 0
+
+Child status: 0
+
+# WARNING:
+# The CPU time of some children was not reported to their father
+# (probably because of a missing or aborted wait()).
+# This 'lost CPU time' is added to the watched process CPU time.
+# lost CPU time (s): 0.38
+# lost CPU user time (s): 0.3
+# lost CPU system time (s): 0.08
+
+Real time (s): 1.00203
+CPU time (s): 0.820027
+CPU user time (s): 0.644021
+CPU system time (s): 0.176006
+CPU usage (%): 81.8367
+Max. virtual memory (cumulated for all children) (KiB): 58944
+
+getrusage(RUSAGE_CHILDREN,...) data:
+user time used= 0.344021
+system time used= 0.096006
+maximum resident set size= 25684
+integral shared memory size= 0
+integral unshared data size= 0
+integral unshared stack size= 0
+page reclaims= 8607
+page faults= 0
+swaps= 0
+block input operations= 0
+block output operations= 8
+messages sent= 0
+messages received= 0
+signals received= 0
+voluntary context switches= 822
+involuntary context switches= 68
+
+runsolver used 0 second user time and 0.028001 second system time
+
+The end
\ No newline at end of file
diff --git a/tests/unittests/runsolver_vsize_exceeded.txt b/tests/unittests/runsolver_vsize_exceeded.txt
new file mode 100644
index 00000000..2e47069d
--- /dev/null
+++ b/tests/unittests/runsolver_vsize_exceeded.txt
@@ -0,0 +1,89 @@
+runsolver Copyright (C) 2010-2013 Olivier ROUSSEL
+
+This is runsolver version 3.3.2 (svn: 1389)
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+command line: runsolver -o /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/branin/smac_2_06_01-dev_1_2014-2-4--13-34-14-689426/2014-2-4--13-59-11-643481_run_instance.out --timestamp --use-pty -W 3600 -C 14400 -M 10 -d 0 python /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/branin/branin.py --fold 0 --folds 1 --params -x2.5 -y7.5
+
+running on 4 cores: 0-3
+
+Enforcing wall clock limit (soft limit, will send SIGTERM then SIGKILL): 3600 seconds
+Enforcing CPUTime limit (soft limit, will send SIGTERM then SIGKILL): 14400 seconds
+Enforcing CPUTime limit (hard limit, will send SIGXCPU): 14430 seconds
+Enforcing VSIZE limit (soft limit, will send SIGTERM then SIGKILL): 10240 KiB
+Enforcing VSIZE limit (hard limit, stack expansion will fail with SIGSEGV, brk() and mmap() will return ENOMEM): 61440 KiB
+Using a pseudo terminal to collect output from the solver
+Current StackSize limit: 8192 KiB
+
+
+[startup+0 s]
+/proc/loadavg: 0.24 0.23 0.28 2/498 27516
+/proc/meminfo: memFree=4949692/8174740 swapFree=0/0
+[pid=27516] ppid=27514 vsize=7764 CPUtime=0 cores=0-3
+/proc/27516/stat : 27516 (python) D 27514 27516 19549 34817 27493 1077944320 720 0 0 0 0 0 0 0 20 0 1 0 60305913 7950336 527 4294967295 134512640 136958856 3213014688 3213000872 3078476836 0 0 16781312 2 4177571357 0 0 17 0 0 0 0 0 0 136965844 137310660 142368768 3213015121 3213015240 3213015240 3213017033 0
+/proc/27516/statm: 1941 527 369 598 0 159 0
+
+
+
+Maximum VSize exceeded: sending SIGTERM then SIGKILL
+
+[startup+0.031982 s]
+/proc/loadavg: 0.24 0.23 0.28 3/499 27517
+/proc/meminfo: memFree=4947708/8174740 swapFree=0/0
+[pid=27516] ppid=27514 vsize=14492 CPUtime=0.01 cores=0-3
+/proc/27516/stat : 27516 (python) D 27514 27516 19549 34817 27493 1077944320 1471 0 0 0 1 0 0 0 20 0 1 0 60305913 14839808 1138 4294967295 134512640 136958856 3213014688 3212920840 3078476836 0 0 16781312 2 4177571357 0 0 17 0 0 0 0 0 0 136965844 137310660 142368768 3213015121 3213015240 3213015240 3213017033 0
+/proc/27516/statm: 3623 1138 632 598 0 495 0
+Current children cumulated CPU time (s) 0.01
+Current children cumulated vsize (KiB) 14492
+
+Sending SIGTERM to process tree (bottom up)
+
+[startup+0.031982 s]
+/proc/loadavg: 0.24 0.23 0.28 3/499 27517
+/proc/meminfo: memFree=4947708/8174740 swapFree=0/0
+[pid=27516] ppid=27514 vsize=14492 CPUtime=0.01 cores=0-3
+/proc/27516/stat : 27516 (python) D 27514 27516 19549 34817 27493 1077944320 1471 0 0 0 1 0 0 0 20 0 1 0 60305913 14839808 1138 4294967295 134512640 136958856 3213014688 3212920840 3078476836 0 0 16781312 2 4177571357 0 0 17 0 0 0 0 0 0 136965844 137310660 142368768 3213015121 3213015240 3213015240 3213017033 0
+/proc/27516/statm: 3623 1138 632 598 0 495 0
+Current children cumulated CPU time (s) 0.01
+Current children cumulated vsize (KiB) 14492
+Sleeping 0 seconds
+
+Sending SIGKILL to process tree (bottom up)
+
+Sending SIGKILL to -27516
+
+Solver just ended. Dumping a history of the last processes samples
+
+Child ended because it received signal 15 (SIGTERM)
+Real time (s): 0.039276
+CPU time (s): 0.016
+CPU user time (s): 0.012
+CPU system time (s): 0.004
+CPU usage (%): 40.7373
+Max. virtual memory (cumulated for all children) (KiB): 14492
+
+getrusage(RUSAGE_CHILDREN,...) data:
+user time used= 0.012
+system time used= 0.004
+maximum resident set size= 4708
+integral shared memory size= 0
+integral unshared data size= 0
+integral unshared stack size= 0
+page reclaims= 1479
+page faults= 0
+swaps= 0
+block input operations= 0
+block output operations= 8
+messages sent= 0
+messages received= 0
+signals received= 0
+voluntary context switches= 62
+involuntary context switches= 4
+
+runsolver used 0.004 second user time and 0.020001 second system time
+
+The end
\ No newline at end of file
diff --git a/tests/unittests/runsolver_wallclock_time_limit.txt b/tests/unittests/runsolver_wallclock_time_limit.txt
new file mode 100644
index 00000000..5a21a1d5
--- /dev/null
+++ b/tests/unittests/runsolver_wallclock_time_limit.txt
@@ -0,0 +1,89 @@
+runsolver Copyright (C) 2010-2013 Olivier ROUSSEL
+
+This is runsolver version 3.3.2 (svn: 1389)
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+command line: runsolver -o /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/branin/smac_2_06_01-dev_1_2014-2-4--13-34-14-689426/2014-2-4--13-52-13-958676_run_instance.out --timestamp --use-pty -W -1 -C 14400 -M 2000 -d 0 python /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/branin/branin.py --fold 0 --folds 1 --params -x2.5 -y7.5
+
+running on 4 cores: 0-3
+
+Enforcing wall clock limit (soft limit, will send SIGTERM then SIGKILL): 4294967295 seconds
+Enforcing CPUTime limit (soft limit, will send SIGTERM then SIGKILL): 14400 seconds
+Enforcing CPUTime limit (hard limit, will send SIGXCPU): 14430 seconds
+Enforcing VSIZE limit (soft limit, will send SIGTERM then SIGKILL): 2048000 KiB
+Enforcing VSIZE limit (hard limit, stack expansion will fail with SIGSEGV, brk() and mmap() will return ENOMEM): 2099200 KiB
+Using a pseudo terminal to collect output from the solver
+Current StackSize limit: 8192 KiB
+
+
+[startup+0 s]
+/proc/loadavg: 0.29 0.34 0.33 3/497 27281
+/proc/meminfo: memFree=4970368/8174740 swapFree=0/0
+[pid=27281] ppid=27279 vsize=8168 CPUtime=0 cores=0-3
+/proc/27281/stat : 27281 (python) R 27279 27281 19549 34817 27258 4202496 796 0 0 0 0 0 0 0 20 0 1 0 60264144 8364032 617 4294967295 134512640 136958856 3214016992 3213989088 134704285 0 0 16781312 2 0 0 0 17 0 0 0 0 0 0 136965844 137310660 150605824 3214022737 3214022856 3214022856 3214024649 0
+/proc/27281/statm: 2042 617 421 598 0 260 0
+
+
+
+Maximum wall clock time exceeded: sending SIGTERM then SIGKILL
+
+[startup+0.053088 s]
+/proc/loadavg: 0.29 0.34 0.33 3/498 27282
+/proc/meminfo: memFree=4969376/8174740 swapFree=0/0
+[pid=27281] ppid=27279 vsize=13972 CPUtime=0.03 cores=0-3
+/proc/27281/stat : 27281 (python) R 27279 27281 19549 34817 27258 4202496 1425 0 0 0 2 1 0 0 20 0 1 0 60264144 14307328 1089 4294967295 134512640 136958856 3214016992 3213927008 3074546142 0 0 16781312 2 0 0 0 17 0 0 0 0 0 0 136965844 137310660 150605824 3214022737 3214022856 3214022856 3214024649 0
+/proc/27281/statm: 3493 1089 610 598 0 460 0
+Current children cumulated CPU time (s) 0.03
+Current children cumulated vsize (KiB) 13972
+
+Sending SIGTERM to process tree (bottom up)
+
+[startup+0.053088 s]
+/proc/loadavg: 0.29 0.34 0.33 3/498 27282
+/proc/meminfo: memFree=4969376/8174740 swapFree=0/0
+[pid=27281] ppid=27279 vsize=13972 CPUtime=0.03 cores=0-3
+/proc/27281/stat : 27281 (python) R 27279 27281 19549 34817 27258 4202496 1425 0 0 0 2 1 0 0 20 0 1 0 60264144 14307328 1089 4294967295 134512640 136958856 3214016992 3213927008 3074546142 0 0 16781312 2 0 0 0 17 0 0 0 0 0 0 136965844 137310660 150605824 3214022737 3214022856 3214022856 3214024649 0
+/proc/27281/statm: 3493 1089 610 598 0 460 0
+Current children cumulated CPU time (s) 0.03
+Current children cumulated vsize (KiB) 13972
+Sleeping 0 seconds
+
+Sending SIGKILL to process tree (bottom up)
+
+Sending SIGKILL to -27281
+
+Solver just ended. Dumping a history of the last processes samples
+
+Child ended because it received signal 15 (SIGTERM)
+Real time (s): 0.066825
+CPU time (s): 0.044002
+CPU user time (s): 0.028001
+CPU system time (s): 0.016001
+CPU usage (%): 65.8466
+Max. virtual memory (cumulated for all children) (KiB): 13972
+
+getrusage(RUSAGE_CHILDREN,...) data:
+user time used= 0.028001
+system time used= 0.016001
+maximum resident set size= 4556
+integral shared memory size= 0
+integral unshared data size= 0
+integral unshared stack size= 0
+page reclaims= 1440
+page faults= 0
+swaps= 0
+block input operations= 0
+block output operations= 8
+messages sent= 0
+messages received= 0
+signals received= 0
+voluntary context switches= 58
+involuntary context switches= 6
+
+runsolver used 0.016001 second user time and 0.020001 second system time
+
+The end
\ No newline at end of file
diff --git a/tests/unittests/search_spaces/__init__.py b/tests/unittests/search_spaces/__init__.py
new file mode 100644
index 00000000..8f0ce6cb
--- /dev/null
+++ b/tests/unittests/search_spaces/__init__.py
@@ -0,0 +1 @@
+__author__ = 'feurerm'
diff --git a/tests/unittests/search_spaces/autoweka.py b/tests/unittests/search_spaces/autoweka.py
new file mode 100644
index 00000000..e6226f6e
--- /dev/null
+++ b/tests/unittests/search_spaces/autoweka.py
@@ -0,0 +1,2028 @@
+from hyperopt import hp
+import math
+
+space = (
+ hp.choice('auto_param_2', [
+ {'attributesearch':'NONE'},
+ (
+ hp.choice('auto_param_4', [
+ {
+ 'attributesearch':'weka.attributeSelection.BestFirst',
+ 'assearch_wasbf__00__01_0_D':hp.choice('assearch_wasbf__00__01_0_D', ['2', '1', '0', ]),
+ 'assearch_wasbf__00__02_1_INT_N':hp.quniform('assearch_wasbf__00__02_1_INT_N', 2.0, 10.0,1.0),
+ 'assearch_wasbf__00__03_2_S':hp.choice('assearch_wasbf__00__03_2_S', ['0', ]),
+ },
+ {
+ 'attributesearch':'weka.attributeSelection.GreedyStepwise',
+ 'assearch_wasgs__01__01_0_C':hp.choice('assearch_wasgs__01__01_0_C', ['REMOVE_PREV', 'REMOVED', ]),
+ 'assearch_wasgs__01__02_1_B':hp.choice('assearch_wasgs__01__02_1_B', ['REMOVE_PREV', 'REMOVED', ]),
+ 'assearch_wasgs__01__03_auto_param_7':hp.choice('assearch_wasgs__01__03_auto_param_7', [
+ {
+ 'assearch_wasgs__01__03__00__00_2_R':'REMOVED',
+ 'assearch_wasgs__01__03__00__01_3_T':hp.uniform('assearch_wasgs__01__03__00__01_3_T', 0.0, 20.0),
+ },
+ {
+ 'assearch_wasgs__01__03__01__00_2_R':'REMOVE_PREV',
+ 'assearch_wasgs__01__03__01__01_4_INT_N':hp.qloguniform('assearch_wasgs__01__03__01__01_4_INT_N', math.log(10.0), math.log(1000.0), 1.0),
+ },
+ ]),
+ },
+ {
+ 'attributesearch':'weka.attributeSelection.Ranker',
+ 'assearch_wasr__02__01_0_T':hp.uniform('assearch_wasr__02__01_0_T', 0.2, 10.0),
+ },
+ ]),
+ hp.choice('auto_param_11', [
+ {
+ 'attributeeval':'weka.attributeSelection.CfsSubsetEval',
+ 'aseval_wascse__00__01_0_M':hp.choice('aseval_wascse__00__01_0_M', ['REMOVE_PREV', 'REMOVED', ]),
+ 'aseval_wascse__00__02_1_L':hp.choice('aseval_wascse__00__02_1_L', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ 'attributeeval':'weka.attributeSelection.CorrelationAttributeEval',
+ },
+ {
+ 'attributeeval':'weka.attributeSelection.GainRatioAttributeEval',
+ },
+ {
+ 'attributeeval':'weka.attributeSelection.InfoGainAttributeEval',
+ 'aseval_wasigae__03__01_0_M':hp.choice('aseval_wasigae__03__01_0_M', ['REMOVE_PREV', 'REMOVED', ]),
+ 'aseval_wasigae__03__02_1_B':hp.choice('aseval_wasigae__03__02_1_B', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ 'attributeeval':'weka.attributeSelection.OneRAttributeEval',
+ 'aseval_wasorae__04__01_0_S':hp.choice('aseval_wasorae__04__01_0_S', ['0', ]),
+ 'aseval_wasorae__04__02_1_F':hp.quniform('aseval_wasorae__04__02_1_F', 2.0, 15.0,1.0),
+ 'aseval_wasorae__04__03_2_D':hp.choice('aseval_wasorae__04__03_2_D', ['REMOVE_PREV', 'REMOVED', ]),
+ 'aseval_wasorae__04__04_3_INT_B':hp.qloguniform('aseval_wasorae__04__04_3_INT_B', math.log(1.0), math.log(64.0), 1.0),
+ },
+ {
+ 'attributeeval':'weka.attributeSelection.PrincipalComponents',
+ 'aseval_waspc__05__01_1_C':hp.choice('aseval_waspc__05__01_1_C', ['REMOVE_PREV', 'REMOVED', ]),
+ 'aseval_waspc__05__02_2_R':hp.uniform('aseval_waspc__05__02_2_R', 0.5, 1.0),
+ 'aseval_waspc__05__03_3_O':hp.choice('aseval_waspc__05__03_3_O', ['REMOVE_PREV', 'REMOVED', ]),
+ 'aseval_waspc__05__04_auto_param_18':hp.choice('aseval_waspc__05__04_auto_param_18', [
+ {
+ 'aseval_waspc__05__04__00__00_num_HIDDEN':'0',
+ 'aseval_waspc__05__04__00__01_1_INT_A':hp.choice('aseval_waspc__05__04__00__01_1_INT_A', ['-1', ]),
+ },
+ {
+ 'aseval_waspc__05__04__01__00_num_HIDDEN':'1',
+ 'aseval_waspc__05__04__01__01_2_INT_A':hp.qloguniform('aseval_waspc__05__04__01__01_2_INT_A', math.log(1.0), math.log(1024.0), 1.0),
+ },
+ ]),
+ },
+ {
+ 'attributeeval':'weka.attributeSelection.ReliefFAttributeEval',
+ 'aseval_wasrfae__06__01_0_INT_K':hp.qloguniform('aseval_wasrfae__06__01_0_INT_K', math.log(2.0), math.log(64.0), 1.0),
+ 'aseval_wasrfae__06__02_auto_param_22':hp.choice('aseval_wasrfae__06__02_auto_param_22', [
+ {
+ 'aseval_wasrfae__06__02__00__00_1_W':'REMOVED',
+ 'aseval_wasrfae__06__02__00__01_2_INT_A':hp.qloguniform('aseval_wasrfae__06__02__00__01_2_INT_A', math.log(1.0), math.log(8.0), 1.0),
+ },
+ {
+ 'aseval_wasrfae__06__02__01__00_1_W':'REMOVE_PREV',
+ },
+ ]),
+ },
+ {
+ 'attributeeval':'weka.attributeSelection.SymmetricalUncertAttributeEval',
+ 'aseval_wassuae__07__01_0_M':hp.choice('aseval_wassuae__07__01_0_M', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ ]),
+ {'attributetime':'900.0'},
+ ),
+ ]),
+ hp.choice('is_base', [
+ hp.choice('base_choice', [
+ {
+ 'targetclass':'weka.classifiers.bayes.BayesNet',
+ '_0_wcbbn__00__01_D':hp.choice('_0_wcbbn__00__01_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcbbn__00__02_Q':hp.choice('_0_wcbbn__00__02_Q', ['weka.classifiers.bayes.net.search.local.TAN', 'weka.classifiers.bayes.net.search.local.HillClimber', 'weka.classifiers.bayes.net.search.local.LAGDHillClimber', 'weka.classifiers.bayes.net.search.local.SimulatedAnnealing', 'weka.classifiers.bayes.net.search.local.TabuSearch', 'weka.classifiers.bayes.net.search.local.K2', ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.bayes.NaiveBayes',
+ '_0_wcbnb__01__01_K':hp.choice('_0_wcbnb__01__01_K', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcbnb__01__02_D':hp.choice('_0_wcbnb__01__02_D', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.bayes.NaiveBayesMultinomial',
+ },
+ {
+ 'targetclass':'weka.classifiers.functions.Logistic',
+ '_0_wcfl__03__01_R':hp.loguniform('_0_wcfl__03__01_R', math.log(1.0E-12), math.log(10.0)),
+ },
+ {
+ 'targetclass':'weka.classifiers.functions.MultilayerPerceptron',
+ '_0_wcfmp__04__01_L':hp.uniform('_0_wcfmp__04__01_L', 0.1, 1.0),
+ '_0_wcfmp__04__02_M':hp.uniform('_0_wcfmp__04__02_M', 0.1, 1.0),
+ '_0_wcfmp__04__03_B':hp.choice('_0_wcfmp__04__03_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcfmp__04__04_H':hp.choice('_0_wcfmp__04__04_H', ['t', 'i', 'o', 'a', ]),
+ '_0_wcfmp__04__05_C':hp.choice('_0_wcfmp__04__05_C', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcfmp__04__06_R':hp.choice('_0_wcfmp__04__06_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcfmp__04__07_D':hp.choice('_0_wcfmp__04__07_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcfmp__04__08_S':hp.choice('_0_wcfmp__04__08_S', ['1', ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.functions.SGD',
+ '_0_wcfsgd__05__01_F':hp.choice('_0_wcfsgd__05__01_F', ['2', '1', '0', ]),
+ '_0_wcfsgd__05__02_L':hp.loguniform('_0_wcfsgd__05__02_L', math.log(1.0E-5), math.log(0.1)),
+ '_0_wcfsgd__05__03_R':hp.loguniform('_0_wcfsgd__05__03_R', math.log(1.0E-12), math.log(10.0)),
+ '_0_wcfsgd__05__04_N':hp.choice('_0_wcfsgd__05__04_N', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcfsgd__05__05_M':hp.choice('_0_wcfsgd__05__05_M', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.functions.SMO',
+ '_0_wcfsmo__06__01_0_C':hp.uniform('_0_wcfsmo__06__01_0_C', 0.5, 1.5),
+ '_0_wcfsmo__06__02_1_N':hp.choice('_0_wcfsmo__06__02_1_N', ['2', '1', '0', ]),
+ '_0_wcfsmo__06__03_2_M':hp.choice('_0_wcfsmo__06__03_2_M', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcfsmo__06__04_5_QUOTE_END':hp.choice('_0_wcfsmo__06__04_5_QUOTE_END', ['REMOVED', ]),
+ '_0_wcfsmo__06__05_auto_param_33':hp.choice('_0_wcfsmo__06__05_auto_param_33', [
+ {
+ '_0_wcfsmo__06__05__00__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.NormalizedPolyKernel',
+ '_0_wcfsmo__06__05__00__01_4_npoly_E':hp.uniform('_0_wcfsmo__06__05__00__01_4_npoly_E', 0.2, 5.0),
+ '_0_wcfsmo__06__05__00__02_4_npoly_L':hp.choice('_0_wcfsmo__06__05__00__02_4_npoly_L', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_0_wcfsmo__06__05__01__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.PolyKernel',
+ '_0_wcfsmo__06__05__01__01_4_poly_E':hp.uniform('_0_wcfsmo__06__05__01__01_4_poly_E', 0.2, 5.0),
+ '_0_wcfsmo__06__05__01__02_4_poly_L':hp.choice('_0_wcfsmo__06__05__01__02_4_poly_L', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_0_wcfsmo__06__05__02__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.Puk',
+ '_0_wcfsmo__06__05__02__01_4_puk_S':hp.uniform('_0_wcfsmo__06__05__02__01_4_puk_S', 0.1, 10.0),
+ '_0_wcfsmo__06__05__02__02_4_puk_O':hp.uniform('_0_wcfsmo__06__05__02__02_4_puk_O', 0.1, 1.0),
+ },
+ {
+ '_0_wcfsmo__06__05__03__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.RBFKernel',
+ '_0_wcfsmo__06__05__03__01_4_rbf_G':hp.loguniform('_0_wcfsmo__06__05__03__01_4_rbf_G', math.log(1.0E-4), math.log(1.0)),
+ },
+ ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.functions.SimpleLogistic',
+ '_0_wcfsl__07__01_S':hp.choice('_0_wcfsl__07__01_S', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcfsl__07__02_A':hp.choice('_0_wcfsl__07__02_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcfsl__07__03_auto_param_39':hp.choice('_0_wcfsl__07__03_auto_param_39', [
+ {
+ '_0_wcfsl__07__03__00__00_W_HIDDEN':'0',
+ '_0_wcfsl__07__03__00__01_1_W':hp.choice('_0_wcfsl__07__03__00__01_1_W', ['0', ]),
+ },
+ {
+ '_0_wcfsl__07__03__01__00_W_HIDDEN':'1',
+ '_0_wcfsl__07__03__01__01_2_W':hp.uniform('_0_wcfsl__07__03__01__01_2_W', 0.0, 1.0),
+ },
+ ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.functions.VotedPerceptron',
+ '_0_wcfvp__08__01_INT_I':hp.quniform('_0_wcfvp__08__01_INT_I', 1.0, 10.0,1.0),
+ '_0_wcfvp__08__02_INT_M':hp.qloguniform('_0_wcfvp__08__02_INT_M', math.log(5000.0), math.log(50000.0), 1.0),
+ '_0_wcfvp__08__03_E':hp.uniform('_0_wcfvp__08__03_E', 0.2, 5.0),
+ },
+ {
+ 'targetclass':'weka.classifiers.lazy.IBk',
+ '_0_wclib__09__01_E':hp.choice('_0_wclib__09__01_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wclib__09__02_INT_K':hp.qloguniform('_0_wclib__09__02_INT_K', math.log(1.0), math.log(64.0), 1.0),
+ '_0_wclib__09__03_X':hp.choice('_0_wclib__09__03_X', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wclib__09__04_F':hp.choice('_0_wclib__09__04_F', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wclib__09__05_I':hp.choice('_0_wclib__09__05_I', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.lazy.KStar',
+ '_0_wclks__10__01_INT_B':hp.quniform('_0_wclks__10__01_INT_B', 1.0, 100.0,1.0),
+ '_0_wclks__10__02_E':hp.choice('_0_wclks__10__02_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wclks__10__03_M':hp.choice('_0_wclks__10__03_M', ['n', 'd', 'm', 'a', ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.rules.DecisionTable',
+ '_0_wcrdt__11__01_E':hp.choice('_0_wcrdt__11__01_E', ['auc', 'rmse', 'mae', 'acc', ]),
+ '_0_wcrdt__11__02_I':hp.choice('_0_wcrdt__11__02_I', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcrdt__11__03_S':hp.choice('_0_wcrdt__11__03_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]),
+ '_0_wcrdt__11__04_X':hp.choice('_0_wcrdt__11__04_X', ['4', '2', '3', '1', ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.rules.JRip',
+ '_0_wcrjr__12__01_N':hp.uniform('_0_wcrjr__12__01_N', 1.0, 5.0),
+ '_0_wcrjr__12__02_E':hp.choice('_0_wcrjr__12__02_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcrjr__12__03_P':hp.choice('_0_wcrjr__12__03_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcrjr__12__04_INT_O':hp.quniform('_0_wcrjr__12__04_INT_O', 1.0, 5.0,1.0),
+ },
+ {
+ 'targetclass':'weka.classifiers.rules.OneR',
+ '_0_wcror__13__01_INT_B':hp.qloguniform('_0_wcror__13__01_INT_B', math.log(1.0), math.log(32.0), 1.0),
+ },
+ {
+ 'targetclass':'weka.classifiers.rules.PART',
+ '_0_wcrpart__14__01_INT_N':hp.quniform('_0_wcrpart__14__01_INT_N', 2.0, 5.0,1.0),
+ '_0_wcrpart__14__02_INT_M':hp.qloguniform('_0_wcrpart__14__02_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_0_wcrpart__14__03_R':hp.choice('_0_wcrpart__14__03_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcrpart__14__04_B':hp.choice('_0_wcrpart__14__04_B', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.rules.ZeroR',
+ },
+ {
+ 'targetclass':'weka.classifiers.trees.DecisionStump',
+ },
+ {
+ 'targetclass':'weka.classifiers.trees.J48',
+ '_0_wctj__17__01_O':hp.choice('_0_wctj__17__01_O', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wctj__17__02_U':hp.choice('_0_wctj__17__02_U', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wctj__17__03_B':hp.choice('_0_wctj__17__03_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wctj__17__04_J':hp.choice('_0_wctj__17__04_J', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wctj__17__05_A':hp.choice('_0_wctj__17__05_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wctj__17__06_S':hp.choice('_0_wctj__17__06_S', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wctj__17__07_INT_M':hp.qloguniform('_0_wctj__17__07_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_0_wctj__17__08_C':hp.uniform('_0_wctj__17__08_C', 0.0, 1.0),
+ },
+ {
+ 'targetclass':'weka.classifiers.trees.LMT',
+ '_0_wctlmt__18__01_B':hp.choice('_0_wctlmt__18__01_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wctlmt__18__02_R':hp.choice('_0_wctlmt__18__02_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wctlmt__18__03_C':hp.choice('_0_wctlmt__18__03_C', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wctlmt__18__04_P':hp.choice('_0_wctlmt__18__04_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wctlmt__18__05_INT_M':hp.qloguniform('_0_wctlmt__18__05_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_0_wctlmt__18__06_A':hp.choice('_0_wctlmt__18__06_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wctlmt__18__07_auto_param_53':hp.choice('_0_wctlmt__18__07_auto_param_53', [
+ {
+ '_0_wctlmt__18__07__00__00_W_HIDDEN':'0',
+ '_0_wctlmt__18__07__00__01_1_W':hp.choice('_0_wctlmt__18__07__00__01_1_W', ['0', ]),
+ },
+ {
+ '_0_wctlmt__18__07__01__00_W_HIDDEN':'1',
+ '_0_wctlmt__18__07__01__01_2_W':hp.uniform('_0_wctlmt__18__07__01__01_2_W', 0.0, 1.0),
+ },
+ ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.trees.REPTree',
+ '_0_wctrept__19__01_INT_M':hp.qloguniform('_0_wctrept__19__01_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_0_wctrept__19__02_V':hp.loguniform('_0_wctrept__19__02_V', math.log(1.0E-5), math.log(0.1)),
+ '_0_wctrept__19__03_P':hp.choice('_0_wctrept__19__03_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wctrept__19__04_auto_param_57':hp.choice('_0_wctrept__19__04_auto_param_57', [
+ {
+ '_0_wctrept__19__04__00__00_depth_HIDDEN':'0',
+ '_0_wctrept__19__04__00__01_1_INT_L':hp.choice('_0_wctrept__19__04__00__01_1_INT_L', ['-1', ]),
+ },
+ {
+ '_0_wctrept__19__04__01__00_depth_HIDDEN':'1',
+ '_0_wctrept__19__04__01__01_2_INT_L':hp.quniform('_0_wctrept__19__04__01__01_2_INT_L', 2.0, 20.0,1.0),
+ },
+ ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.trees.RandomForest',
+ '_0_wctrf__20__01_INT_I':hp.qloguniform('_0_wctrf__20__01_INT_I', math.log(2.0), math.log(256.0), 1.0),
+ '_0_wctrf__20__02_auto_param_61':hp.choice('_0_wctrf__20__02_auto_param_61', [
+ {
+ '_0_wctrf__20__02__00__00_features_HIDDEN':'0',
+ '_0_wctrf__20__02__00__01_1_INT_K':hp.choice('_0_wctrf__20__02__00__01_1_INT_K', ['1', ]),
+ },
+ {
+ '_0_wctrf__20__02__01__00_features_HIDDEN':'1',
+ '_0_wctrf__20__02__01__01_2_INT_K':hp.qloguniform('_0_wctrf__20__02__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0),
+ },
+ ]),
+ '_0_wctrf__20__03_auto_param_64':hp.choice('_0_wctrf__20__03_auto_param_64', [
+ {
+ '_0_wctrf__20__03__00__00_depth_HIDDEN':'0',
+ '_0_wctrf__20__03__00__01_1_INT_depth':hp.choice('_0_wctrf__20__03__00__01_1_INT_depth', ['1', ]),
+ },
+ {
+ '_0_wctrf__20__03__01__00_depth_HIDDEN':'1',
+ '_0_wctrf__20__03__01__01_2_INT_depth':hp.quniform('_0_wctrf__20__03__01__01_2_INT_depth', 2.0, 20.0,1.0),
+ },
+ ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.trees.RandomTree',
+ '_0_wctrt__21__01_INT_M':hp.qloguniform('_0_wctrt__21__01_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_0_wctrt__21__02_U':hp.choice('_0_wctrt__21__02_U', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wctrt__21__03_auto_param_68':hp.choice('_0_wctrt__21__03_auto_param_68', [
+ {
+ '_0_wctrt__21__03__00__00_features_HIDDEN':'0',
+ '_0_wctrt__21__03__00__01_1_INT_K':hp.choice('_0_wctrt__21__03__00__01_1_INT_K', ['0', ]),
+ },
+ {
+ '_0_wctrt__21__03__01__00_features_HIDDEN':'1',
+ '_0_wctrt__21__03__01__01_2_INT_K':hp.qloguniform('_0_wctrt__21__03__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0),
+ },
+ ]),
+ '_0_wctrt__21__04_auto_param_71':hp.choice('_0_wctrt__21__04_auto_param_71', [
+ {
+ '_0_wctrt__21__04__00__00_depth_HIDDEN':'0',
+ '_0_wctrt__21__04__00__01_1_INT_depth':hp.choice('_0_wctrt__21__04__00__01_1_INT_depth', ['0', ]),
+ },
+ {
+ '_0_wctrt__21__04__01__00_depth_HIDDEN':'1',
+ '_0_wctrt__21__04__01__01_2_INT_depth':hp.quniform('_0_wctrt__21__04__01__01_2_INT_depth', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_0_wctrt__21__05_auto_param_74':hp.choice('_0_wctrt__21__05_auto_param_74', [
+ {
+ '_0_wctrt__21__05__00__00_back_HIDDEN':'0',
+ '_0_wctrt__21__05__00__01_1_INT_N':hp.choice('_0_wctrt__21__05__00__01_1_INT_N', ['0', ]),
+ },
+ {
+ '_0_wctrt__21__05__01__00_back_HIDDEN':'1',
+ '_0_wctrt__21__05__01__01_2_INT_N':hp.quniform('_0_wctrt__21__05__01__01_2_INT_N', 2.0, 5.0,1.0),
+ },
+ ]),
+ },
+ ]),
+ hp.choice('is_meta', [
+ (
+ hp.choice('meta_choice', [
+ {
+ 'targetclass':'weka.classifiers.lazy.LWL',
+ '_0_wcllwl__00__01_K':hp.choice('_0_wcllwl__00__01_K', ['120', '10', '30', '60', '90', '-1', ]),
+ '_0_wcllwl__00__02_U':hp.choice('_0_wcllwl__00__02_U', ['4', '1', '2', '3', '0', ]),
+ '_0_wcllwl__00__03_A':hp.choice('_0_wcllwl__00__03_A', ['weka.core.neighboursearch.LinearNNSearch', ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.meta.AdaBoostM1',
+ '_0_wcmabm__01__01_INT_I':hp.qloguniform('_0_wcmabm__01__01_INT_I', math.log(2.0), math.log(128.0), 1.0),
+ '_0_wcmabm__01__02_Q':hp.choice('_0_wcmabm__01__02_Q', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcmabm__01__03_S':hp.choice('_0_wcmabm__01__03_S', ['1', ]),
+ '_0_wcmabm__01__04_auto_param_80':hp.choice('_0_wcmabm__01__04_auto_param_80', [
+ {
+ '_0_wcmabm__01__04__00__00_p_HIDDEN':'0',
+ '_0_wcmabm__01__04__00__01_1_P':hp.choice('_0_wcmabm__01__04__00__01_1_P', ['100', ]),
+ },
+ {
+ '_0_wcmabm__01__04__01__00_p_HIDDEN':'1',
+ '_0_wcmabm__01__04__01__01_2_INT_P':hp.quniform('_0_wcmabm__01__04__01__01_2_INT_P', 50.0, 100.0,1.0),
+ },
+ ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.meta.AttributeSelectedClassifier',
+ '_0_wcmasc__02__01_S':hp.choice('_0_wcmasc__02__01_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]),
+ '_0_wcmasc__02__02_E':hp.choice('_0_wcmasc__02__02_E', ['weka.attributeSelection.GainRatioAttributeEval', 'weka.attributeSelection.WrapperSubsetEval', 'weka.attributeSelection.OneRAttributeEval', 'weka.attributeSelection.InfoGainAttributeEval', 'weka.attributeSelection.HoldOutSubsetEvaluator', 'weka.attributeSelection.CfsSubsetEval', ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.meta.Bagging',
+ '_0_wcmb__03__01_INT_P':hp.quniform('_0_wcmb__03__01_INT_P', 10.0, 200.0,1.0),
+ '_0_wcmb__03__02_INT_I':hp.qloguniform('_0_wcmb__03__02_INT_I', math.log(2.0), math.log(128.0), 1.0),
+ '_0_wcmb__03__03_S':hp.choice('_0_wcmb__03__03_S', ['1', ]),
+ '_0_wcmb__03__04_O':hp.choice('_0_wcmb__03__04_O', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.meta.ClassificationViaRegression',
+ },
+ {
+ 'targetclass':'weka.classifiers.meta.LogitBoost',
+ '_0_wcmlb__05__01_INT_I':hp.qloguniform('_0_wcmlb__05__01_INT_I', math.log(2.0), math.log(128.0), 1.0),
+ '_0_wcmlb__05__02_INT_R':hp.quniform('_0_wcmlb__05__02_INT_R', 1.0, 5.0,1.0),
+ '_0_wcmlb__05__03_Q':hp.choice('_0_wcmlb__05__03_Q', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcmlb__05__04_L':hp.choice('_0_wcmlb__05__04_L', ['1e50', ]),
+ '_0_wcmlb__05__05_S':hp.choice('_0_wcmlb__05__05_S', ['1', ]),
+ '_0_wcmlb__05__06_auto_param_87':hp.choice('_0_wcmlb__05__06_auto_param_87', [
+ {
+ '_0_wcmlb__05__06__00__00_h_HIDDEN':'0',
+ '_0_wcmlb__05__06__00__01_1_H':hp.choice('_0_wcmlb__05__06__00__01_1_H', ['1', ]),
+ },
+ {
+ '_0_wcmlb__05__06__01__00_h_HIDDEN':'1',
+ '_0_wcmlb__05__06__01__01_2_H':hp.uniform('_0_wcmlb__05__06__01__01_2_H', 0.0, 1.0),
+ },
+ ]),
+ '_0_wcmlb__05__07_auto_param_90':hp.choice('_0_wcmlb__05__07_auto_param_90', [
+ {
+ '_0_wcmlb__05__07__00__00_f_HIDDEN':'0',
+ '_0_wcmlb__05__07__00__01_1_F':hp.choice('_0_wcmlb__05__07__00__01_1_F', ['0', ]),
+ },
+ {
+ '_0_wcmlb__05__07__01__00_f_HIDDEN':'1',
+ '_0_wcmlb__05__07__01__01_2_INT_F':hp.quniform('_0_wcmlb__05__07__01__01_2_INT_F', 1.0, 5.0,1.0),
+ },
+ ]),
+ '_0_wcmlb__05__08_auto_param_93':hp.choice('_0_wcmlb__05__08_auto_param_93', [
+ {
+ '_0_wcmlb__05__08__00__00_p_HIDDEN':'0',
+ '_0_wcmlb__05__08__00__01_1_P':hp.choice('_0_wcmlb__05__08__00__01_1_P', ['100', ]),
+ },
+ {
+ '_0_wcmlb__05__08__01__00_p_HIDDEN':'1',
+ '_0_wcmlb__05__08__01__01_2_INT_P':hp.quniform('_0_wcmlb__05__08__01__01_2_INT_P', 50.0, 100.0,1.0),
+ },
+ ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.meta.MultiClassClassifier',
+ '_0_wcmmcc__06__01_M':hp.choice('_0_wcmmcc__06__01_M', ['3', '1', '2', '0', ]),
+ '_0_wcmmcc__06__02_R':hp.uniform('_0_wcmmcc__06__02_R', 0.5, 4.0),
+ '_0_wcmmcc__06__03_P':hp.choice('_0_wcmmcc__06__03_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_0_wcmmcc__06__04_S':hp.choice('_0_wcmmcc__06__04_S', ['1', ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.meta.RandomCommittee',
+ '_0_wcmrc__07__01_INT_I':hp.qloguniform('_0_wcmrc__07__01_INT_I', math.log(2.0), math.log(64.0), 1.0),
+ '_0_wcmrc__07__02_S':hp.choice('_0_wcmrc__07__02_S', ['1', ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.meta.RandomSubSpace',
+ '_0_wcmrss__08__01_INT_I':hp.qloguniform('_0_wcmrss__08__01_INT_I', math.log(2.0), math.log(64.0), 1.0),
+ '_0_wcmrss__08__02_P':hp.uniform('_0_wcmrss__08__02_P', 0.1, 1.0),
+ '_0_wcmrss__08__03_S':hp.choice('_0_wcmrss__08__03_S', ['1', ]),
+ },
+ ]),
+ hp.choice('meta_base_choice', [
+ {
+ '_1_wcbbn_W':'weka.classifiers.bayes.BayesNet',
+ '_1_wcbbn_W-':'REMOVED',
+ '_1_wcbbn_W_00__02_D':hp.choice('_1_wcbbn_W_00__02_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wcbbn_W_00__03_Q':hp.choice('_1_wcbbn_W_00__03_Q', ['weka.classifiers.bayes.net.search.local.TAN', 'weka.classifiers.bayes.net.search.local.HillClimber', 'weka.classifiers.bayes.net.search.local.LAGDHillClimber', 'weka.classifiers.bayes.net.search.local.SimulatedAnnealing', 'weka.classifiers.bayes.net.search.local.TabuSearch', 'weka.classifiers.bayes.net.search.local.K2', ]),
+ },
+ {
+ '_1_wcbnb_W':'weka.classifiers.bayes.NaiveBayes',
+ '_1_wcbnb_W-':'REMOVED',
+ '_1_wcbnb_W_01__02_K':hp.choice('_1_wcbnb_W_01__02_K', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wcbnb_W_01__03_D':hp.choice('_1_wcbnb_W_01__03_D', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_wcbnbm_W':'weka.classifiers.bayes.NaiveBayesMultinomial',
+ '_1_wcbnbm_W-':'REMOVED',
+ },
+ {
+ '_1_wcfl_W':'weka.classifiers.functions.Logistic',
+ '_1_wcfl_W-':'REMOVED',
+ '_1_wcfl_W_03__02_R':hp.loguniform('_1_wcfl_W_03__02_R', math.log(1.0E-12), math.log(10.0)),
+ },
+ {
+ '_1_wcfmp_W':'weka.classifiers.functions.MultilayerPerceptron',
+ '_1_wcfmp_W-':'REMOVED',
+ '_1_wcfmp_W_04__02_L':hp.uniform('_1_wcfmp_W_04__02_L', 0.1, 1.0),
+ '_1_wcfmp_W_04__03_M':hp.uniform('_1_wcfmp_W_04__03_M', 0.1, 1.0),
+ '_1_wcfmp_W_04__04_B':hp.choice('_1_wcfmp_W_04__04_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wcfmp_W_04__05_H':hp.choice('_1_wcfmp_W_04__05_H', ['t', 'i', 'o', 'a', ]),
+ '_1_wcfmp_W_04__06_C':hp.choice('_1_wcfmp_W_04__06_C', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wcfmp_W_04__07_R':hp.choice('_1_wcfmp_W_04__07_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wcfmp_W_04__08_D':hp.choice('_1_wcfmp_W_04__08_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wcfmp_W_04__09_S':hp.choice('_1_wcfmp_W_04__09_S', ['1', ]),
+ },
+ {
+ '_1_wcfsgd_W':'weka.classifiers.functions.SGD',
+ '_1_wcfsgd_W-':'REMOVED',
+ '_1_wcfsgd_W_05__02_F':hp.choice('_1_wcfsgd_W_05__02_F', ['2', '1', '0', ]),
+ '_1_wcfsgd_W_05__03_L':hp.loguniform('_1_wcfsgd_W_05__03_L', math.log(1.0E-5), math.log(0.1)),
+ '_1_wcfsgd_W_05__04_R':hp.loguniform('_1_wcfsgd_W_05__04_R', math.log(1.0E-12), math.log(10.0)),
+ '_1_wcfsgd_W_05__05_N':hp.choice('_1_wcfsgd_W_05__05_N', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wcfsgd_W_05__06_M':hp.choice('_1_wcfsgd_W_05__06_M', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_wcfsmo_W':'weka.classifiers.functions.SMO',
+ '_1_wcfsmo_W-':'REMOVED',
+ '_1_wcfsmo_W_06__02_0_C':hp.uniform('_1_wcfsmo_W_06__02_0_C', 0.5, 1.5),
+ '_1_wcfsmo_W_06__03_1_N':hp.choice('_1_wcfsmo_W_06__03_1_N', ['2', '1', '0', ]),
+ '_1_wcfsmo_W_06__04_2_M':hp.choice('_1_wcfsmo_W_06__04_2_M', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wcfsmo_W_06__05_5_QUOTE_END':hp.choice('_1_wcfsmo_W_06__05_5_QUOTE_END', ['REMOVED', ]),
+ '_1_wcfsmo_W_06__06_auto_param_106':hp.choice('_1_wcfsmo_W_06__06_auto_param_106', [
+ {
+ '_1_wcfsmo_W_06__06__00__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.NormalizedPolyKernel',
+ '_1_wcfsmo_W_06__06__00__01_4_npoly_E':hp.uniform('_1_wcfsmo_W_06__06__00__01_4_npoly_E', 0.2, 5.0),
+ '_1_wcfsmo_W_06__06__00__02_4_npoly_L':hp.choice('_1_wcfsmo_W_06__06__00__02_4_npoly_L', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_wcfsmo_W_06__06__01__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.PolyKernel',
+ '_1_wcfsmo_W_06__06__01__01_4_poly_E':hp.uniform('_1_wcfsmo_W_06__06__01__01_4_poly_E', 0.2, 5.0),
+ '_1_wcfsmo_W_06__06__01__02_4_poly_L':hp.choice('_1_wcfsmo_W_06__06__01__02_4_poly_L', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_wcfsmo_W_06__06__02__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.Puk',
+ '_1_wcfsmo_W_06__06__02__01_4_puk_S':hp.uniform('_1_wcfsmo_W_06__06__02__01_4_puk_S', 0.1, 10.0),
+ '_1_wcfsmo_W_06__06__02__02_4_puk_O':hp.uniform('_1_wcfsmo_W_06__06__02__02_4_puk_O', 0.1, 1.0),
+ },
+ {
+ '_1_wcfsmo_W_06__06__03__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.RBFKernel',
+ '_1_wcfsmo_W_06__06__03__01_4_rbf_G':hp.loguniform('_1_wcfsmo_W_06__06__03__01_4_rbf_G', math.log(1.0E-4), math.log(1.0)),
+ },
+ ]),
+ },
+ {
+ '_1_wcfsl_W':'weka.classifiers.functions.SimpleLogistic',
+ '_1_wcfsl_W-':'REMOVED',
+ '_1_wcfsl_W_07__02_S':hp.choice('_1_wcfsl_W_07__02_S', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wcfsl_W_07__03_A':hp.choice('_1_wcfsl_W_07__03_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wcfsl_W_07__04_auto_param_112':hp.choice('_1_wcfsl_W_07__04_auto_param_112', [
+ {
+ '_1_wcfsl_W_07__04__00__00_W_HIDDEN':'0',
+ '_1_wcfsl_W_07__04__00__01_1_W':hp.choice('_1_wcfsl_W_07__04__00__01_1_W', ['0', ]),
+ },
+ {
+ '_1_wcfsl_W_07__04__01__00_W_HIDDEN':'1',
+ '_1_wcfsl_W_07__04__01__01_2_W':hp.uniform('_1_wcfsl_W_07__04__01__01_2_W', 0.0, 1.0),
+ },
+ ]),
+ },
+ {
+ '_1_wcfvp_W':'weka.classifiers.functions.VotedPerceptron',
+ '_1_wcfvp_W-':'REMOVED',
+ '_1_wcfvp_W_08__02_INT_I':hp.quniform('_1_wcfvp_W_08__02_INT_I', 1.0, 10.0,1.0),
+ '_1_wcfvp_W_08__03_INT_M':hp.qloguniform('_1_wcfvp_W_08__03_INT_M', math.log(5000.0), math.log(50000.0), 1.0),
+ '_1_wcfvp_W_08__04_E':hp.uniform('_1_wcfvp_W_08__04_E', 0.2, 5.0),
+ },
+ {
+ '_1_wclib_W':'weka.classifiers.lazy.IBk',
+ '_1_wclib_W-':'REMOVED',
+ '_1_wclib_W_09__02_E':hp.choice('_1_wclib_W_09__02_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wclib_W_09__03_INT_K':hp.qloguniform('_1_wclib_W_09__03_INT_K', math.log(1.0), math.log(64.0), 1.0),
+ '_1_wclib_W_09__04_X':hp.choice('_1_wclib_W_09__04_X', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wclib_W_09__05_F':hp.choice('_1_wclib_W_09__05_F', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wclib_W_09__06_I':hp.choice('_1_wclib_W_09__06_I', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_wclks_W':'weka.classifiers.lazy.KStar',
+ '_1_wclks_W-':'REMOVED',
+ '_1_wclks_W_10__02_INT_B':hp.quniform('_1_wclks_W_10__02_INT_B', 1.0, 100.0,1.0),
+ '_1_wclks_W_10__03_E':hp.choice('_1_wclks_W_10__03_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wclks_W_10__04_M':hp.choice('_1_wclks_W_10__04_M', ['n', 'd', 'm', 'a', ]),
+ },
+ {
+ '_1_wcrdt_W':'weka.classifiers.rules.DecisionTable',
+ '_1_wcrdt_W-':'REMOVED',
+ '_1_wcrdt_W_11__02_E':hp.choice('_1_wcrdt_W_11__02_E', ['auc', 'rmse', 'mae', 'acc', ]),
+ '_1_wcrdt_W_11__03_I':hp.choice('_1_wcrdt_W_11__03_I', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wcrdt_W_11__04_S':hp.choice('_1_wcrdt_W_11__04_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]),
+ '_1_wcrdt_W_11__05_X':hp.choice('_1_wcrdt_W_11__05_X', ['4', '2', '3', '1', ]),
+ },
+ {
+ '_1_wcrjr_W':'weka.classifiers.rules.JRip',
+ '_1_wcrjr_W-':'REMOVED',
+ '_1_wcrjr_W_12__02_N':hp.uniform('_1_wcrjr_W_12__02_N', 1.0, 5.0),
+ '_1_wcrjr_W_12__03_E':hp.choice('_1_wcrjr_W_12__03_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wcrjr_W_12__04_P':hp.choice('_1_wcrjr_W_12__04_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wcrjr_W_12__05_INT_O':hp.quniform('_1_wcrjr_W_12__05_INT_O', 1.0, 5.0,1.0),
+ },
+ {
+ '_1_wcror_W':'weka.classifiers.rules.OneR',
+ '_1_wcror_W-':'REMOVED',
+ '_1_wcror_W_13__02_INT_B':hp.qloguniform('_1_wcror_W_13__02_INT_B', math.log(1.0), math.log(32.0), 1.0),
+ },
+ {
+ '_1_wcrpart_W':'weka.classifiers.rules.PART',
+ '_1_wcrpart_W-':'REMOVED',
+ '_1_wcrpart_W_14__02_INT_N':hp.quniform('_1_wcrpart_W_14__02_INT_N', 2.0, 5.0,1.0),
+ '_1_wcrpart_W_14__03_INT_M':hp.qloguniform('_1_wcrpart_W_14__03_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_wcrpart_W_14__04_R':hp.choice('_1_wcrpart_W_14__04_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wcrpart_W_14__05_B':hp.choice('_1_wcrpart_W_14__05_B', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_wcrzr_W':'weka.classifiers.rules.ZeroR',
+ '_1_wcrzr_W-':'REMOVED',
+ },
+ {
+ '_1_wctds_W':'weka.classifiers.trees.DecisionStump',
+ '_1_wctds_W-':'REMOVED',
+ },
+ {
+ '_1_wctj_W':'weka.classifiers.trees.J48',
+ '_1_wctj_W-':'REMOVED',
+ '_1_wctj_W_17__02_O':hp.choice('_1_wctj_W_17__02_O', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wctj_W_17__03_U':hp.choice('_1_wctj_W_17__03_U', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wctj_W_17__04_B':hp.choice('_1_wctj_W_17__04_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wctj_W_17__05_J':hp.choice('_1_wctj_W_17__05_J', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wctj_W_17__06_A':hp.choice('_1_wctj_W_17__06_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wctj_W_17__07_S':hp.choice('_1_wctj_W_17__07_S', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wctj_W_17__08_INT_M':hp.qloguniform('_1_wctj_W_17__08_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_wctj_W_17__09_C':hp.uniform('_1_wctj_W_17__09_C', 0.0, 1.0),
+ },
+ {
+ '_1_wctlmt_W':'weka.classifiers.trees.LMT',
+ '_1_wctlmt_W-':'REMOVED',
+ '_1_wctlmt_W_18__02_B':hp.choice('_1_wctlmt_W_18__02_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wctlmt_W_18__03_R':hp.choice('_1_wctlmt_W_18__03_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wctlmt_W_18__04_C':hp.choice('_1_wctlmt_W_18__04_C', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wctlmt_W_18__05_P':hp.choice('_1_wctlmt_W_18__05_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wctlmt_W_18__06_INT_M':hp.qloguniform('_1_wctlmt_W_18__06_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_wctlmt_W_18__07_A':hp.choice('_1_wctlmt_W_18__07_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wctlmt_W_18__08_auto_param_126':hp.choice('_1_wctlmt_W_18__08_auto_param_126', [
+ {
+ '_1_wctlmt_W_18__08__00__00_W_HIDDEN':'0',
+ '_1_wctlmt_W_18__08__00__01_1_W':hp.choice('_1_wctlmt_W_18__08__00__01_1_W', ['0', ]),
+ },
+ {
+ '_1_wctlmt_W_18__08__01__00_W_HIDDEN':'1',
+ '_1_wctlmt_W_18__08__01__01_2_W':hp.uniform('_1_wctlmt_W_18__08__01__01_2_W', 0.0, 1.0),
+ },
+ ]),
+ },
+ {
+ '_1_wctrept_W':'weka.classifiers.trees.REPTree',
+ '_1_wctrept_W-':'REMOVED',
+ '_1_wctrept_W_19__02_INT_M':hp.qloguniform('_1_wctrept_W_19__02_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_wctrept_W_19__03_V':hp.loguniform('_1_wctrept_W_19__03_V', math.log(1.0E-5), math.log(0.1)),
+ '_1_wctrept_W_19__04_P':hp.choice('_1_wctrept_W_19__04_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wctrept_W_19__05_auto_param_130':hp.choice('_1_wctrept_W_19__05_auto_param_130', [
+ {
+ '_1_wctrept_W_19__05__00__00_depth_HIDDEN':'0',
+ '_1_wctrept_W_19__05__00__01_1_INT_L':hp.choice('_1_wctrept_W_19__05__00__01_1_INT_L', ['-1', ]),
+ },
+ {
+ '_1_wctrept_W_19__05__01__00_depth_HIDDEN':'1',
+ '_1_wctrept_W_19__05__01__01_2_INT_L':hp.quniform('_1_wctrept_W_19__05__01__01_2_INT_L', 2.0, 20.0,1.0),
+ },
+ ]),
+ },
+ {
+ '_1_wctrf_W':'weka.classifiers.trees.RandomForest',
+ '_1_wctrf_W-':'REMOVED',
+ '_1_wctrf_W_20__02_INT_I':hp.qloguniform('_1_wctrf_W_20__02_INT_I', math.log(2.0), math.log(256.0), 1.0),
+ '_1_wctrf_W_20__03_auto_param_134':hp.choice('_1_wctrf_W_20__03_auto_param_134', [
+ {
+ '_1_wctrf_W_20__03__00__00_features_HIDDEN':'0',
+ '_1_wctrf_W_20__03__00__01_1_INT_K':hp.choice('_1_wctrf_W_20__03__00__01_1_INT_K', ['1', ]),
+ },
+ {
+ '_1_wctrf_W_20__03__01__00_features_HIDDEN':'1',
+ '_1_wctrf_W_20__03__01__01_2_INT_K':hp.qloguniform('_1_wctrf_W_20__03__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0),
+ },
+ ]),
+ '_1_wctrf_W_20__04_auto_param_137':hp.choice('_1_wctrf_W_20__04_auto_param_137', [
+ {
+ '_1_wctrf_W_20__04__00__00_depth_HIDDEN':'0',
+ '_1_wctrf_W_20__04__00__01_1_INT_depth':hp.choice('_1_wctrf_W_20__04__00__01_1_INT_depth', ['1', ]),
+ },
+ {
+ '_1_wctrf_W_20__04__01__00_depth_HIDDEN':'1',
+ '_1_wctrf_W_20__04__01__01_2_INT_depth':hp.quniform('_1_wctrf_W_20__04__01__01_2_INT_depth', 2.0, 20.0,1.0),
+ },
+ ]),
+ },
+ {
+ '_1_wctrt_W':'weka.classifiers.trees.RandomTree',
+ '_1_wctrt_W-':'REMOVED',
+ '_1_wctrt_W_21__02_INT_M':hp.qloguniform('_1_wctrt_W_21__02_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_wctrt_W_21__03_U':hp.choice('_1_wctrt_W_21__03_U', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_wctrt_W_21__04_auto_param_141':hp.choice('_1_wctrt_W_21__04_auto_param_141', [
+ {
+ '_1_wctrt_W_21__04__00__00_features_HIDDEN':'0',
+ '_1_wctrt_W_21__04__00__01_1_INT_K':hp.choice('_1_wctrt_W_21__04__00__01_1_INT_K', ['0', ]),
+ },
+ {
+ '_1_wctrt_W_21__04__01__00_features_HIDDEN':'1',
+ '_1_wctrt_W_21__04__01__01_2_INT_K':hp.qloguniform('_1_wctrt_W_21__04__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0),
+ },
+ ]),
+ '_1_wctrt_W_21__05_auto_param_144':hp.choice('_1_wctrt_W_21__05_auto_param_144', [
+ {
+ '_1_wctrt_W_21__05__00__00_depth_HIDDEN':'0',
+ '_1_wctrt_W_21__05__00__01_1_INT_depth':hp.choice('_1_wctrt_W_21__05__00__01_1_INT_depth', ['0', ]),
+ },
+ {
+ '_1_wctrt_W_21__05__01__00_depth_HIDDEN':'1',
+ '_1_wctrt_W_21__05__01__01_2_INT_depth':hp.quniform('_1_wctrt_W_21__05__01__01_2_INT_depth', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_wctrt_W_21__06_auto_param_147':hp.choice('_1_wctrt_W_21__06_auto_param_147', [
+ {
+ '_1_wctrt_W_21__06__00__00_back_HIDDEN':'0',
+ '_1_wctrt_W_21__06__00__01_1_INT_N':hp.choice('_1_wctrt_W_21__06__00__01_1_INT_N', ['0', ]),
+ },
+ {
+ '_1_wctrt_W_21__06__01__00_back_HIDDEN':'1',
+ '_1_wctrt_W_21__06__01__01_2_INT_N':hp.quniform('_1_wctrt_W_21__06__01__01_2_INT_N', 2.0, 5.0,1.0),
+ },
+ ]),
+ },
+ ]),
+ ),
+ (
+ hp.choice('ensemble_choice', [
+ {
+ 'targetclass':'weka.classifiers.meta.Stacking',
+ '_0_wcms__00__01_X':hp.choice('_0_wcms__00__01_X', ['10', ]),
+ '_0_wcms__00__02_S':hp.choice('_0_wcms__00__02_S', ['1', ]),
+ },
+ {
+ 'targetclass':'weka.classifiers.meta.Vote',
+ '_0_wcmv__01__01_R':hp.choice('_0_wcmv__01__01_R', ['MAX', 'PROD', 'MAJ', 'MIN', 'AVG', ]),
+ '_0_wcmv__01__02_S':hp.choice('_0_wcmv__01__02_S', ['1', ]),
+ },
+ ]),
+ (
+ hp.choice('ensemble_base_choice', [
+ {
+ '_1_00_wcbbn_0_QUOTE_START_B':'weka.classifiers.bayes.BayesNet',
+ '_1_00_wcbbn_1__00__01_D':hp.choice('_1_00_wcbbn_1__00__01_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcbbn_1__00__02_Q':hp.choice('_1_00_wcbbn_1__00__02_Q', ['weka.classifiers.bayes.net.search.local.TAN', 'weka.classifiers.bayes.net.search.local.HillClimber', 'weka.classifiers.bayes.net.search.local.LAGDHillClimber', 'weka.classifiers.bayes.net.search.local.SimulatedAnnealing', 'weka.classifiers.bayes.net.search.local.TabuSearch', 'weka.classifiers.bayes.net.search.local.K2', ]),
+ '_1_00_wcbbn_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wcbnb_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayes',
+ '_1_00_wcbnb_1__01__01_K':hp.choice('_1_00_wcbnb_1__01__01_K', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcbnb_1__01__02_D':hp.choice('_1_00_wcbnb_1__01__02_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcbnb_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wcbnbm_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayesMultinomial',
+ '_1_00_wcbnbm_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wcfl_0_QUOTE_START_B':'weka.classifiers.functions.Logistic',
+ '_1_00_wcfl_1__03__01_R':hp.loguniform('_1_00_wcfl_1__03__01_R', math.log(1.0E-12), math.log(10.0)),
+ '_1_00_wcfl_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wcfmp_0_QUOTE_START_B':'weka.classifiers.functions.MultilayerPerceptron',
+ '_1_00_wcfmp_1__04__01_L':hp.uniform('_1_00_wcfmp_1__04__01_L', 0.1, 1.0),
+ '_1_00_wcfmp_1__04__02_M':hp.uniform('_1_00_wcfmp_1__04__02_M', 0.1, 1.0),
+ '_1_00_wcfmp_1__04__03_B':hp.choice('_1_00_wcfmp_1__04__03_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcfmp_1__04__04_H':hp.choice('_1_00_wcfmp_1__04__04_H', ['t', 'i', 'o', 'a', ]),
+ '_1_00_wcfmp_1__04__05_C':hp.choice('_1_00_wcfmp_1__04__05_C', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcfmp_1__04__06_R':hp.choice('_1_00_wcfmp_1__04__06_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcfmp_1__04__07_D':hp.choice('_1_00_wcfmp_1__04__07_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcfmp_1__04__08_S':hp.choice('_1_00_wcfmp_1__04__08_S', ['1', ]),
+ '_1_00_wcfmp_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wcfsgd_0_QUOTE_START_B':'weka.classifiers.functions.SGD',
+ '_1_00_wcfsgd_1__05__01_F':hp.choice('_1_00_wcfsgd_1__05__01_F', ['2', '1', '0', ]),
+ '_1_00_wcfsgd_1__05__02_L':hp.loguniform('_1_00_wcfsgd_1__05__02_L', math.log(1.0E-5), math.log(0.1)),
+ '_1_00_wcfsgd_1__05__03_R':hp.loguniform('_1_00_wcfsgd_1__05__03_R', math.log(1.0E-12), math.log(10.0)),
+ '_1_00_wcfsgd_1__05__04_N':hp.choice('_1_00_wcfsgd_1__05__04_N', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcfsgd_1__05__05_M':hp.choice('_1_00_wcfsgd_1__05__05_M', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcfsgd_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wcfsmo_0_QUOTE_START_B':'weka.classifiers.functions.SMO',
+ '_1_00_wcfsmo_1__06__01_0_C':hp.uniform('_1_00_wcfsmo_1__06__01_0_C', 0.5, 1.5),
+ '_1_00_wcfsmo_1__06__02_1_N':hp.choice('_1_00_wcfsmo_1__06__02_1_N', ['2', '1', '0', ]),
+ '_1_00_wcfsmo_1__06__03_2_M':hp.choice('_1_00_wcfsmo_1__06__03_2_M', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcfsmo_1__06__04_5_QUOTE_END':hp.choice('_1_00_wcfsmo_1__06__04_5_QUOTE_END', ['REMOVED', ]),
+ '_1_00_wcfsmo_1__06__05_auto_param_161':hp.choice('_1_00_wcfsmo_1__06__05_auto_param_161', [
+ {
+ '_1_00_wcfsmo_1__06__05__00__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.NormalizedPolyKernel',
+ '_1_00_wcfsmo_1__06__05__00__01_4_npoly_E':hp.uniform('_1_00_wcfsmo_1__06__05__00__01_4_npoly_E', 0.2, 5.0),
+ '_1_00_wcfsmo_1__06__05__00__02_4_npoly_L':hp.choice('_1_00_wcfsmo_1__06__05__00__02_4_npoly_L', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_00_wcfsmo_1__06__05__01__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.PolyKernel',
+ '_1_00_wcfsmo_1__06__05__01__01_4_poly_E':hp.uniform('_1_00_wcfsmo_1__06__05__01__01_4_poly_E', 0.2, 5.0),
+ '_1_00_wcfsmo_1__06__05__01__02_4_poly_L':hp.choice('_1_00_wcfsmo_1__06__05__01__02_4_poly_L', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_00_wcfsmo_1__06__05__02__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.Puk',
+ '_1_00_wcfsmo_1__06__05__02__01_4_puk_S':hp.uniform('_1_00_wcfsmo_1__06__05__02__01_4_puk_S', 0.1, 10.0),
+ '_1_00_wcfsmo_1__06__05__02__02_4_puk_O':hp.uniform('_1_00_wcfsmo_1__06__05__02__02_4_puk_O', 0.1, 1.0),
+ },
+ {
+ '_1_00_wcfsmo_1__06__05__03__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.RBFKernel',
+ '_1_00_wcfsmo_1__06__05__03__01_4_rbf_G':hp.loguniform('_1_00_wcfsmo_1__06__05__03__01_4_rbf_G', math.log(1.0E-4), math.log(1.0)),
+ },
+ ]),
+ '_1_00_wcfsmo_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wcfsl_0_QUOTE_START_B':'weka.classifiers.functions.SimpleLogistic',
+ '_1_00_wcfsl_1__07__01_S':hp.choice('_1_00_wcfsl_1__07__01_S', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcfsl_1__07__02_A':hp.choice('_1_00_wcfsl_1__07__02_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcfsl_1__07__03_auto_param_167':hp.choice('_1_00_wcfsl_1__07__03_auto_param_167', [
+ {
+ '_1_00_wcfsl_1__07__03__00__00_W_HIDDEN':'0',
+ '_1_00_wcfsl_1__07__03__00__01_1_W':hp.choice('_1_00_wcfsl_1__07__03__00__01_1_W', ['0', ]),
+ },
+ {
+ '_1_00_wcfsl_1__07__03__01__00_W_HIDDEN':'1',
+ '_1_00_wcfsl_1__07__03__01__01_2_W':hp.uniform('_1_00_wcfsl_1__07__03__01__01_2_W', 0.0, 1.0),
+ },
+ ]),
+ '_1_00_wcfsl_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wcfvp_0_QUOTE_START_B':'weka.classifiers.functions.VotedPerceptron',
+ '_1_00_wcfvp_1__08__01_INT_I':hp.quniform('_1_00_wcfvp_1__08__01_INT_I', 1.0, 10.0,1.0),
+ '_1_00_wcfvp_1__08__02_INT_M':hp.qloguniform('_1_00_wcfvp_1__08__02_INT_M', math.log(5000.0), math.log(50000.0), 1.0),
+ '_1_00_wcfvp_1__08__03_E':hp.uniform('_1_00_wcfvp_1__08__03_E', 0.2, 5.0),
+ '_1_00_wcfvp_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wclib_0_QUOTE_START_B':'weka.classifiers.lazy.IBk',
+ '_1_00_wclib_1__09__01_E':hp.choice('_1_00_wclib_1__09__01_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wclib_1__09__02_INT_K':hp.qloguniform('_1_00_wclib_1__09__02_INT_K', math.log(1.0), math.log(64.0), 1.0),
+ '_1_00_wclib_1__09__03_X':hp.choice('_1_00_wclib_1__09__03_X', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wclib_1__09__04_F':hp.choice('_1_00_wclib_1__09__04_F', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wclib_1__09__05_I':hp.choice('_1_00_wclib_1__09__05_I', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wclib_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wclks_0_QUOTE_START_B':'weka.classifiers.lazy.KStar',
+ '_1_00_wclks_1__10__01_INT_B':hp.quniform('_1_00_wclks_1__10__01_INT_B', 1.0, 100.0,1.0),
+ '_1_00_wclks_1__10__02_E':hp.choice('_1_00_wclks_1__10__02_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wclks_1__10__03_M':hp.choice('_1_00_wclks_1__10__03_M', ['n', 'd', 'm', 'a', ]),
+ '_1_00_wclks_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wcrdt_0_QUOTE_START_B':'weka.classifiers.rules.DecisionTable',
+ '_1_00_wcrdt_1__11__01_E':hp.choice('_1_00_wcrdt_1__11__01_E', ['auc', 'rmse', 'mae', 'acc', ]),
+ '_1_00_wcrdt_1__11__02_I':hp.choice('_1_00_wcrdt_1__11__02_I', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcrdt_1__11__03_S':hp.choice('_1_00_wcrdt_1__11__03_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]),
+ '_1_00_wcrdt_1__11__04_X':hp.choice('_1_00_wcrdt_1__11__04_X', ['4', '2', '3', '1', ]),
+ '_1_00_wcrdt_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wcrjr_0_QUOTE_START_B':'weka.classifiers.rules.JRip',
+ '_1_00_wcrjr_1__12__01_N':hp.uniform('_1_00_wcrjr_1__12__01_N', 1.0, 5.0),
+ '_1_00_wcrjr_1__12__02_E':hp.choice('_1_00_wcrjr_1__12__02_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcrjr_1__12__03_P':hp.choice('_1_00_wcrjr_1__12__03_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcrjr_1__12__04_INT_O':hp.quniform('_1_00_wcrjr_1__12__04_INT_O', 1.0, 5.0,1.0),
+ '_1_00_wcrjr_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wcror_0_QUOTE_START_B':'weka.classifiers.rules.OneR',
+ '_1_00_wcror_1__13__01_INT_B':hp.qloguniform('_1_00_wcror_1__13__01_INT_B', math.log(1.0), math.log(32.0), 1.0),
+ '_1_00_wcror_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wcrpart_0_QUOTE_START_B':'weka.classifiers.rules.PART',
+ '_1_00_wcrpart_1__14__01_INT_N':hp.quniform('_1_00_wcrpart_1__14__01_INT_N', 2.0, 5.0,1.0),
+ '_1_00_wcrpart_1__14__02_INT_M':hp.qloguniform('_1_00_wcrpart_1__14__02_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_00_wcrpart_1__14__03_R':hp.choice('_1_00_wcrpart_1__14__03_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcrpart_1__14__04_B':hp.choice('_1_00_wcrpart_1__14__04_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wcrpart_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wcrzr_0_QUOTE_START_B':'weka.classifiers.rules.ZeroR',
+ '_1_00_wcrzr_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wctds_0_QUOTE_START_B':'weka.classifiers.trees.DecisionStump',
+ '_1_00_wctds_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wctj_0_QUOTE_START_B':'weka.classifiers.trees.J48',
+ '_1_00_wctj_1__17__01_O':hp.choice('_1_00_wctj_1__17__01_O', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wctj_1__17__02_U':hp.choice('_1_00_wctj_1__17__02_U', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wctj_1__17__03_B':hp.choice('_1_00_wctj_1__17__03_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wctj_1__17__04_J':hp.choice('_1_00_wctj_1__17__04_J', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wctj_1__17__05_A':hp.choice('_1_00_wctj_1__17__05_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wctj_1__17__06_S':hp.choice('_1_00_wctj_1__17__06_S', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wctj_1__17__07_INT_M':hp.qloguniform('_1_00_wctj_1__17__07_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_00_wctj_1__17__08_C':hp.uniform('_1_00_wctj_1__17__08_C', 0.0, 1.0),
+ '_1_00_wctj_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wctlmt_0_QUOTE_START_B':'weka.classifiers.trees.LMT',
+ '_1_00_wctlmt_1__18__01_B':hp.choice('_1_00_wctlmt_1__18__01_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wctlmt_1__18__02_R':hp.choice('_1_00_wctlmt_1__18__02_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wctlmt_1__18__03_C':hp.choice('_1_00_wctlmt_1__18__03_C', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wctlmt_1__18__04_P':hp.choice('_1_00_wctlmt_1__18__04_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wctlmt_1__18__05_INT_M':hp.qloguniform('_1_00_wctlmt_1__18__05_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_00_wctlmt_1__18__06_A':hp.choice('_1_00_wctlmt_1__18__06_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wctlmt_1__18__07_auto_param_181':hp.choice('_1_00_wctlmt_1__18__07_auto_param_181', [
+ {
+ '_1_00_wctlmt_1__18__07__00__00_W_HIDDEN':'0',
+ '_1_00_wctlmt_1__18__07__00__01_1_W':hp.choice('_1_00_wctlmt_1__18__07__00__01_1_W', ['0', ]),
+ },
+ {
+ '_1_00_wctlmt_1__18__07__01__00_W_HIDDEN':'1',
+ '_1_00_wctlmt_1__18__07__01__01_2_W':hp.uniform('_1_00_wctlmt_1__18__07__01__01_2_W', 0.0, 1.0),
+ },
+ ]),
+ '_1_00_wctlmt_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wctrept_0_QUOTE_START_B':'weka.classifiers.trees.REPTree',
+ '_1_00_wctrept_1__19__01_INT_M':hp.qloguniform('_1_00_wctrept_1__19__01_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_00_wctrept_1__19__02_V':hp.loguniform('_1_00_wctrept_1__19__02_V', math.log(1.0E-5), math.log(0.1)),
+ '_1_00_wctrept_1__19__03_P':hp.choice('_1_00_wctrept_1__19__03_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wctrept_1__19__04_auto_param_185':hp.choice('_1_00_wctrept_1__19__04_auto_param_185', [
+ {
+ '_1_00_wctrept_1__19__04__00__00_depth_HIDDEN':'0',
+ '_1_00_wctrept_1__19__04__00__01_1_INT_L':hp.choice('_1_00_wctrept_1__19__04__00__01_1_INT_L', ['-1', ]),
+ },
+ {
+ '_1_00_wctrept_1__19__04__01__00_depth_HIDDEN':'1',
+ '_1_00_wctrept_1__19__04__01__01_2_INT_L':hp.quniform('_1_00_wctrept_1__19__04__01__01_2_INT_L', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_00_wctrept_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wctrf_0_QUOTE_START_B':'weka.classifiers.trees.RandomForest',
+ '_1_00_wctrf_1__20__01_INT_I':hp.qloguniform('_1_00_wctrf_1__20__01_INT_I', math.log(2.0), math.log(256.0), 1.0),
+ '_1_00_wctrf_1__20__02_auto_param_189':hp.choice('_1_00_wctrf_1__20__02_auto_param_189', [
+ {
+ '_1_00_wctrf_1__20__02__00__00_features_HIDDEN':'0',
+ '_1_00_wctrf_1__20__02__00__01_1_INT_K':hp.choice('_1_00_wctrf_1__20__02__00__01_1_INT_K', ['1', ]),
+ },
+ {
+ '_1_00_wctrf_1__20__02__01__00_features_HIDDEN':'1',
+ '_1_00_wctrf_1__20__02__01__01_2_INT_K':hp.qloguniform('_1_00_wctrf_1__20__02__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0),
+ },
+ ]),
+ '_1_00_wctrf_1__20__03_auto_param_192':hp.choice('_1_00_wctrf_1__20__03_auto_param_192', [
+ {
+ '_1_00_wctrf_1__20__03__00__00_depth_HIDDEN':'0',
+ '_1_00_wctrf_1__20__03__00__01_1_INT_depth':hp.choice('_1_00_wctrf_1__20__03__00__01_1_INT_depth', ['1', ]),
+ },
+ {
+ '_1_00_wctrf_1__20__03__01__00_depth_HIDDEN':'1',
+ '_1_00_wctrf_1__20__03__01__01_2_INT_depth':hp.quniform('_1_00_wctrf_1__20__03__01__01_2_INT_depth', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_00_wctrf_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_00_wctrt_0_QUOTE_START_B':'weka.classifiers.trees.RandomTree',
+ '_1_00_wctrt_1__21__01_INT_M':hp.qloguniform('_1_00_wctrt_1__21__01_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_00_wctrt_1__21__02_U':hp.choice('_1_00_wctrt_1__21__02_U', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_00_wctrt_1__21__03_auto_param_196':hp.choice('_1_00_wctrt_1__21__03_auto_param_196', [
+ {
+ '_1_00_wctrt_1__21__03__00__00_features_HIDDEN':'0',
+ '_1_00_wctrt_1__21__03__00__01_1_INT_K':hp.choice('_1_00_wctrt_1__21__03__00__01_1_INT_K', ['0', ]),
+ },
+ {
+ '_1_00_wctrt_1__21__03__01__00_features_HIDDEN':'1',
+ '_1_00_wctrt_1__21__03__01__01_2_INT_K':hp.qloguniform('_1_00_wctrt_1__21__03__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0),
+ },
+ ]),
+ '_1_00_wctrt_1__21__04_auto_param_199':hp.choice('_1_00_wctrt_1__21__04_auto_param_199', [
+ {
+ '_1_00_wctrt_1__21__04__00__00_depth_HIDDEN':'0',
+ '_1_00_wctrt_1__21__04__00__01_1_INT_depth':hp.choice('_1_00_wctrt_1__21__04__00__01_1_INT_depth', ['0', ]),
+ },
+ {
+ '_1_00_wctrt_1__21__04__01__00_depth_HIDDEN':'1',
+ '_1_00_wctrt_1__21__04__01__01_2_INT_depth':hp.quniform('_1_00_wctrt_1__21__04__01__01_2_INT_depth', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_00_wctrt_1__21__05_auto_param_202':hp.choice('_1_00_wctrt_1__21__05_auto_param_202', [
+ {
+ '_1_00_wctrt_1__21__05__00__00_back_HIDDEN':'0',
+ '_1_00_wctrt_1__21__05__00__01_1_INT_N':hp.choice('_1_00_wctrt_1__21__05__00__01_1_INT_N', ['0', ]),
+ },
+ {
+ '_1_00_wctrt_1__21__05__01__00_back_HIDDEN':'1',
+ '_1_00_wctrt_1__21__05__01__01_2_INT_N':hp.quniform('_1_00_wctrt_1__21__05__01__01_2_INT_N', 2.0, 5.0,1.0),
+ },
+ ]),
+ '_1_00_wctrt_2_QUOTE_END':'REMOVED',
+ },
+ ]),
+ hp.choice('auto_param_205', [
+ (
+ ),
+ (
+ hp.choice('auto_param_208', [
+ {
+ '_1_01_wcbbn_0_QUOTE_START_B':'weka.classifiers.bayes.BayesNet',
+ '_1_01_wcbbn_1__00__01_D':hp.choice('_1_01_wcbbn_1__00__01_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcbbn_1__00__02_Q':hp.choice('_1_01_wcbbn_1__00__02_Q', ['weka.classifiers.bayes.net.search.local.TAN', 'weka.classifiers.bayes.net.search.local.HillClimber', 'weka.classifiers.bayes.net.search.local.LAGDHillClimber', 'weka.classifiers.bayes.net.search.local.SimulatedAnnealing', 'weka.classifiers.bayes.net.search.local.TabuSearch', 'weka.classifiers.bayes.net.search.local.K2', ]),
+ '_1_01_wcbbn_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wcbnb_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayes',
+ '_1_01_wcbnb_1__01__01_K':hp.choice('_1_01_wcbnb_1__01__01_K', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcbnb_1__01__02_D':hp.choice('_1_01_wcbnb_1__01__02_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcbnb_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wcbnbm_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayesMultinomial',
+ '_1_01_wcbnbm_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wcfl_0_QUOTE_START_B':'weka.classifiers.functions.Logistic',
+ '_1_01_wcfl_1__03__01_R':hp.loguniform('_1_01_wcfl_1__03__01_R', math.log(1.0E-12), math.log(10.0)),
+ '_1_01_wcfl_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wcfmp_0_QUOTE_START_B':'weka.classifiers.functions.MultilayerPerceptron',
+ '_1_01_wcfmp_1__04__01_L':hp.uniform('_1_01_wcfmp_1__04__01_L', 0.1, 1.0),
+ '_1_01_wcfmp_1__04__02_M':hp.uniform('_1_01_wcfmp_1__04__02_M', 0.1, 1.0),
+ '_1_01_wcfmp_1__04__03_B':hp.choice('_1_01_wcfmp_1__04__03_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcfmp_1__04__04_H':hp.choice('_1_01_wcfmp_1__04__04_H', ['t', 'i', 'o', 'a', ]),
+ '_1_01_wcfmp_1__04__05_C':hp.choice('_1_01_wcfmp_1__04__05_C', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcfmp_1__04__06_R':hp.choice('_1_01_wcfmp_1__04__06_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcfmp_1__04__07_D':hp.choice('_1_01_wcfmp_1__04__07_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcfmp_1__04__08_S':hp.choice('_1_01_wcfmp_1__04__08_S', ['1', ]),
+ '_1_01_wcfmp_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wcfsgd_0_QUOTE_START_B':'weka.classifiers.functions.SGD',
+ '_1_01_wcfsgd_1__05__01_F':hp.choice('_1_01_wcfsgd_1__05__01_F', ['2', '1', '0', ]),
+ '_1_01_wcfsgd_1__05__02_L':hp.loguniform('_1_01_wcfsgd_1__05__02_L', math.log(1.0E-5), math.log(0.1)),
+ '_1_01_wcfsgd_1__05__03_R':hp.loguniform('_1_01_wcfsgd_1__05__03_R', math.log(1.0E-12), math.log(10.0)),
+ '_1_01_wcfsgd_1__05__04_N':hp.choice('_1_01_wcfsgd_1__05__04_N', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcfsgd_1__05__05_M':hp.choice('_1_01_wcfsgd_1__05__05_M', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcfsgd_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wcfsmo_0_QUOTE_START_B':'weka.classifiers.functions.SMO',
+ '_1_01_wcfsmo_1__06__01_0_C':hp.uniform('_1_01_wcfsmo_1__06__01_0_C', 0.5, 1.5),
+ '_1_01_wcfsmo_1__06__02_1_N':hp.choice('_1_01_wcfsmo_1__06__02_1_N', ['2', '1', '0', ]),
+ '_1_01_wcfsmo_1__06__03_2_M':hp.choice('_1_01_wcfsmo_1__06__03_2_M', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcfsmo_1__06__04_5_QUOTE_END':hp.choice('_1_01_wcfsmo_1__06__04_5_QUOTE_END', ['REMOVED', ]),
+ '_1_01_wcfsmo_1__06__05_auto_param_216':hp.choice('_1_01_wcfsmo_1__06__05_auto_param_216', [
+ {
+ '_1_01_wcfsmo_1__06__05__00__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.NormalizedPolyKernel',
+ '_1_01_wcfsmo_1__06__05__00__01_4_npoly_E':hp.uniform('_1_01_wcfsmo_1__06__05__00__01_4_npoly_E', 0.2, 5.0),
+ '_1_01_wcfsmo_1__06__05__00__02_4_npoly_L':hp.choice('_1_01_wcfsmo_1__06__05__00__02_4_npoly_L', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_01_wcfsmo_1__06__05__01__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.PolyKernel',
+ '_1_01_wcfsmo_1__06__05__01__01_4_poly_E':hp.uniform('_1_01_wcfsmo_1__06__05__01__01_4_poly_E', 0.2, 5.0),
+ '_1_01_wcfsmo_1__06__05__01__02_4_poly_L':hp.choice('_1_01_wcfsmo_1__06__05__01__02_4_poly_L', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_01_wcfsmo_1__06__05__02__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.Puk',
+ '_1_01_wcfsmo_1__06__05__02__01_4_puk_S':hp.uniform('_1_01_wcfsmo_1__06__05__02__01_4_puk_S', 0.1, 10.0),
+ '_1_01_wcfsmo_1__06__05__02__02_4_puk_O':hp.uniform('_1_01_wcfsmo_1__06__05__02__02_4_puk_O', 0.1, 1.0),
+ },
+ {
+ '_1_01_wcfsmo_1__06__05__03__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.RBFKernel',
+ '_1_01_wcfsmo_1__06__05__03__01_4_rbf_G':hp.loguniform('_1_01_wcfsmo_1__06__05__03__01_4_rbf_G', math.log(1.0E-4), math.log(1.0)),
+ },
+ ]),
+ '_1_01_wcfsmo_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wcfsl_0_QUOTE_START_B':'weka.classifiers.functions.SimpleLogistic',
+ '_1_01_wcfsl_1__07__01_S':hp.choice('_1_01_wcfsl_1__07__01_S', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcfsl_1__07__02_A':hp.choice('_1_01_wcfsl_1__07__02_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcfsl_1__07__03_auto_param_222':hp.choice('_1_01_wcfsl_1__07__03_auto_param_222', [
+ {
+ '_1_01_wcfsl_1__07__03__00__00_W_HIDDEN':'0',
+ '_1_01_wcfsl_1__07__03__00__01_1_W':hp.choice('_1_01_wcfsl_1__07__03__00__01_1_W', ['0', ]),
+ },
+ {
+ '_1_01_wcfsl_1__07__03__01__00_W_HIDDEN':'1',
+ '_1_01_wcfsl_1__07__03__01__01_2_W':hp.uniform('_1_01_wcfsl_1__07__03__01__01_2_W', 0.0, 1.0),
+ },
+ ]),
+ '_1_01_wcfsl_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wcfvp_0_QUOTE_START_B':'weka.classifiers.functions.VotedPerceptron',
+ '_1_01_wcfvp_1__08__01_INT_I':hp.quniform('_1_01_wcfvp_1__08__01_INT_I', 1.0, 10.0,1.0),
+ '_1_01_wcfvp_1__08__02_INT_M':hp.qloguniform('_1_01_wcfvp_1__08__02_INT_M', math.log(5000.0), math.log(50000.0), 1.0),
+ '_1_01_wcfvp_1__08__03_E':hp.uniform('_1_01_wcfvp_1__08__03_E', 0.2, 5.0),
+ '_1_01_wcfvp_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wclib_0_QUOTE_START_B':'weka.classifiers.lazy.IBk',
+ '_1_01_wclib_1__09__01_E':hp.choice('_1_01_wclib_1__09__01_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wclib_1__09__02_INT_K':hp.qloguniform('_1_01_wclib_1__09__02_INT_K', math.log(1.0), math.log(64.0), 1.0),
+ '_1_01_wclib_1__09__03_X':hp.choice('_1_01_wclib_1__09__03_X', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wclib_1__09__04_F':hp.choice('_1_01_wclib_1__09__04_F', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wclib_1__09__05_I':hp.choice('_1_01_wclib_1__09__05_I', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wclib_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wclks_0_QUOTE_START_B':'weka.classifiers.lazy.KStar',
+ '_1_01_wclks_1__10__01_INT_B':hp.quniform('_1_01_wclks_1__10__01_INT_B', 1.0, 100.0,1.0),
+ '_1_01_wclks_1__10__02_E':hp.choice('_1_01_wclks_1__10__02_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wclks_1__10__03_M':hp.choice('_1_01_wclks_1__10__03_M', ['n', 'd', 'm', 'a', ]),
+ '_1_01_wclks_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wcrdt_0_QUOTE_START_B':'weka.classifiers.rules.DecisionTable',
+ '_1_01_wcrdt_1__11__01_E':hp.choice('_1_01_wcrdt_1__11__01_E', ['auc', 'rmse', 'mae', 'acc', ]),
+ '_1_01_wcrdt_1__11__02_I':hp.choice('_1_01_wcrdt_1__11__02_I', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcrdt_1__11__03_S':hp.choice('_1_01_wcrdt_1__11__03_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]),
+ '_1_01_wcrdt_1__11__04_X':hp.choice('_1_01_wcrdt_1__11__04_X', ['4', '2', '3', '1', ]),
+ '_1_01_wcrdt_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wcrjr_0_QUOTE_START_B':'weka.classifiers.rules.JRip',
+ '_1_01_wcrjr_1__12__01_N':hp.uniform('_1_01_wcrjr_1__12__01_N', 1.0, 5.0),
+ '_1_01_wcrjr_1__12__02_E':hp.choice('_1_01_wcrjr_1__12__02_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcrjr_1__12__03_P':hp.choice('_1_01_wcrjr_1__12__03_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcrjr_1__12__04_INT_O':hp.quniform('_1_01_wcrjr_1__12__04_INT_O', 1.0, 5.0,1.0),
+ '_1_01_wcrjr_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wcror_0_QUOTE_START_B':'weka.classifiers.rules.OneR',
+ '_1_01_wcror_1__13__01_INT_B':hp.qloguniform('_1_01_wcror_1__13__01_INT_B', math.log(1.0), math.log(32.0), 1.0),
+ '_1_01_wcror_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wcrpart_0_QUOTE_START_B':'weka.classifiers.rules.PART',
+ '_1_01_wcrpart_1__14__01_INT_N':hp.quniform('_1_01_wcrpart_1__14__01_INT_N', 2.0, 5.0,1.0),
+ '_1_01_wcrpart_1__14__02_INT_M':hp.qloguniform('_1_01_wcrpart_1__14__02_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_01_wcrpart_1__14__03_R':hp.choice('_1_01_wcrpart_1__14__03_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcrpart_1__14__04_B':hp.choice('_1_01_wcrpart_1__14__04_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wcrpart_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wcrzr_0_QUOTE_START_B':'weka.classifiers.rules.ZeroR',
+ '_1_01_wcrzr_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wctds_0_QUOTE_START_B':'weka.classifiers.trees.DecisionStump',
+ '_1_01_wctds_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wctj_0_QUOTE_START_B':'weka.classifiers.trees.J48',
+ '_1_01_wctj_1__17__01_O':hp.choice('_1_01_wctj_1__17__01_O', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wctj_1__17__02_U':hp.choice('_1_01_wctj_1__17__02_U', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wctj_1__17__03_B':hp.choice('_1_01_wctj_1__17__03_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wctj_1__17__04_J':hp.choice('_1_01_wctj_1__17__04_J', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wctj_1__17__05_A':hp.choice('_1_01_wctj_1__17__05_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wctj_1__17__06_S':hp.choice('_1_01_wctj_1__17__06_S', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wctj_1__17__07_INT_M':hp.qloguniform('_1_01_wctj_1__17__07_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_01_wctj_1__17__08_C':hp.uniform('_1_01_wctj_1__17__08_C', 0.0, 1.0),
+ '_1_01_wctj_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wctlmt_0_QUOTE_START_B':'weka.classifiers.trees.LMT',
+ '_1_01_wctlmt_1__18__01_B':hp.choice('_1_01_wctlmt_1__18__01_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wctlmt_1__18__02_R':hp.choice('_1_01_wctlmt_1__18__02_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wctlmt_1__18__03_C':hp.choice('_1_01_wctlmt_1__18__03_C', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wctlmt_1__18__04_P':hp.choice('_1_01_wctlmt_1__18__04_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wctlmt_1__18__05_INT_M':hp.qloguniform('_1_01_wctlmt_1__18__05_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_01_wctlmt_1__18__06_A':hp.choice('_1_01_wctlmt_1__18__06_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wctlmt_1__18__07_auto_param_236':hp.choice('_1_01_wctlmt_1__18__07_auto_param_236', [
+ {
+ '_1_01_wctlmt_1__18__07__00__00_W_HIDDEN':'0',
+ '_1_01_wctlmt_1__18__07__00__01_1_W':hp.choice('_1_01_wctlmt_1__18__07__00__01_1_W', ['0', ]),
+ },
+ {
+ '_1_01_wctlmt_1__18__07__01__00_W_HIDDEN':'1',
+ '_1_01_wctlmt_1__18__07__01__01_2_W':hp.uniform('_1_01_wctlmt_1__18__07__01__01_2_W', 0.0, 1.0),
+ },
+ ]),
+ '_1_01_wctlmt_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wctrept_0_QUOTE_START_B':'weka.classifiers.trees.REPTree',
+ '_1_01_wctrept_1__19__01_INT_M':hp.qloguniform('_1_01_wctrept_1__19__01_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_01_wctrept_1__19__02_V':hp.loguniform('_1_01_wctrept_1__19__02_V', math.log(1.0E-5), math.log(0.1)),
+ '_1_01_wctrept_1__19__03_P':hp.choice('_1_01_wctrept_1__19__03_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wctrept_1__19__04_auto_param_240':hp.choice('_1_01_wctrept_1__19__04_auto_param_240', [
+ {
+ '_1_01_wctrept_1__19__04__00__00_depth_HIDDEN':'0',
+ '_1_01_wctrept_1__19__04__00__01_1_INT_L':hp.choice('_1_01_wctrept_1__19__04__00__01_1_INT_L', ['-1', ]),
+ },
+ {
+ '_1_01_wctrept_1__19__04__01__00_depth_HIDDEN':'1',
+ '_1_01_wctrept_1__19__04__01__01_2_INT_L':hp.quniform('_1_01_wctrept_1__19__04__01__01_2_INT_L', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_01_wctrept_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wctrf_0_QUOTE_START_B':'weka.classifiers.trees.RandomForest',
+ '_1_01_wctrf_1__20__01_INT_I':hp.qloguniform('_1_01_wctrf_1__20__01_INT_I', math.log(2.0), math.log(256.0), 1.0),
+ '_1_01_wctrf_1__20__02_auto_param_244':hp.choice('_1_01_wctrf_1__20__02_auto_param_244', [
+ {
+ '_1_01_wctrf_1__20__02__00__00_features_HIDDEN':'0',
+ '_1_01_wctrf_1__20__02__00__01_1_INT_K':hp.choice('_1_01_wctrf_1__20__02__00__01_1_INT_K', ['1', ]),
+ },
+ {
+ '_1_01_wctrf_1__20__02__01__00_features_HIDDEN':'1',
+ '_1_01_wctrf_1__20__02__01__01_2_INT_K':hp.qloguniform('_1_01_wctrf_1__20__02__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0),
+ },
+ ]),
+ '_1_01_wctrf_1__20__03_auto_param_247':hp.choice('_1_01_wctrf_1__20__03_auto_param_247', [
+ {
+ '_1_01_wctrf_1__20__03__00__00_depth_HIDDEN':'0',
+ '_1_01_wctrf_1__20__03__00__01_1_INT_depth':hp.choice('_1_01_wctrf_1__20__03__00__01_1_INT_depth', ['1', ]),
+ },
+ {
+ '_1_01_wctrf_1__20__03__01__00_depth_HIDDEN':'1',
+ '_1_01_wctrf_1__20__03__01__01_2_INT_depth':hp.quniform('_1_01_wctrf_1__20__03__01__01_2_INT_depth', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_01_wctrf_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_01_wctrt_0_QUOTE_START_B':'weka.classifiers.trees.RandomTree',
+ '_1_01_wctrt_1__21__01_INT_M':hp.qloguniform('_1_01_wctrt_1__21__01_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_01_wctrt_1__21__02_U':hp.choice('_1_01_wctrt_1__21__02_U', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_01_wctrt_1__21__03_auto_param_251':hp.choice('_1_01_wctrt_1__21__03_auto_param_251', [
+ {
+ '_1_01_wctrt_1__21__03__00__00_features_HIDDEN':'0',
+ '_1_01_wctrt_1__21__03__00__01_1_INT_K':hp.choice('_1_01_wctrt_1__21__03__00__01_1_INT_K', ['0', ]),
+ },
+ {
+ '_1_01_wctrt_1__21__03__01__00_features_HIDDEN':'1',
+ '_1_01_wctrt_1__21__03__01__01_2_INT_K':hp.qloguniform('_1_01_wctrt_1__21__03__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0),
+ },
+ ]),
+ '_1_01_wctrt_1__21__04_auto_param_254':hp.choice('_1_01_wctrt_1__21__04_auto_param_254', [
+ {
+ '_1_01_wctrt_1__21__04__00__00_depth_HIDDEN':'0',
+ '_1_01_wctrt_1__21__04__00__01_1_INT_depth':hp.choice('_1_01_wctrt_1__21__04__00__01_1_INT_depth', ['0', ]),
+ },
+ {
+ '_1_01_wctrt_1__21__04__01__00_depth_HIDDEN':'1',
+ '_1_01_wctrt_1__21__04__01__01_2_INT_depth':hp.quniform('_1_01_wctrt_1__21__04__01__01_2_INT_depth', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_01_wctrt_1__21__05_auto_param_257':hp.choice('_1_01_wctrt_1__21__05_auto_param_257', [
+ {
+ '_1_01_wctrt_1__21__05__00__00_back_HIDDEN':'0',
+ '_1_01_wctrt_1__21__05__00__01_1_INT_N':hp.choice('_1_01_wctrt_1__21__05__00__01_1_INT_N', ['0', ]),
+ },
+ {
+ '_1_01_wctrt_1__21__05__01__00_back_HIDDEN':'1',
+ '_1_01_wctrt_1__21__05__01__01_2_INT_N':hp.quniform('_1_01_wctrt_1__21__05__01__01_2_INT_N', 2.0, 5.0,1.0),
+ },
+ ]),
+ '_1_01_wctrt_2_QUOTE_END':'REMOVED',
+ },
+ ]),
+ hp.choice('auto_param_260', [
+ (
+ ),
+ (
+ hp.choice('auto_param_263', [
+ {
+ '_1_02_wcbbn_0_QUOTE_START_B':'weka.classifiers.bayes.BayesNet',
+ '_1_02_wcbbn_1__00__01_D':hp.choice('_1_02_wcbbn_1__00__01_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcbbn_1__00__02_Q':hp.choice('_1_02_wcbbn_1__00__02_Q', ['weka.classifiers.bayes.net.search.local.TAN', 'weka.classifiers.bayes.net.search.local.HillClimber', 'weka.classifiers.bayes.net.search.local.LAGDHillClimber', 'weka.classifiers.bayes.net.search.local.SimulatedAnnealing', 'weka.classifiers.bayes.net.search.local.TabuSearch', 'weka.classifiers.bayes.net.search.local.K2', ]),
+ '_1_02_wcbbn_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wcbnb_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayes',
+ '_1_02_wcbnb_1__01__01_K':hp.choice('_1_02_wcbnb_1__01__01_K', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcbnb_1__01__02_D':hp.choice('_1_02_wcbnb_1__01__02_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcbnb_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wcbnbm_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayesMultinomial',
+ '_1_02_wcbnbm_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wcfl_0_QUOTE_START_B':'weka.classifiers.functions.Logistic',
+ '_1_02_wcfl_1__03__01_R':hp.loguniform('_1_02_wcfl_1__03__01_R', math.log(1.0E-12), math.log(10.0)),
+ '_1_02_wcfl_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wcfmp_0_QUOTE_START_B':'weka.classifiers.functions.MultilayerPerceptron',
+ '_1_02_wcfmp_1__04__01_L':hp.uniform('_1_02_wcfmp_1__04__01_L', 0.1, 1.0),
+ '_1_02_wcfmp_1__04__02_M':hp.uniform('_1_02_wcfmp_1__04__02_M', 0.1, 1.0),
+ '_1_02_wcfmp_1__04__03_B':hp.choice('_1_02_wcfmp_1__04__03_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcfmp_1__04__04_H':hp.choice('_1_02_wcfmp_1__04__04_H', ['t', 'i', 'o', 'a', ]),
+ '_1_02_wcfmp_1__04__05_C':hp.choice('_1_02_wcfmp_1__04__05_C', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcfmp_1__04__06_R':hp.choice('_1_02_wcfmp_1__04__06_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcfmp_1__04__07_D':hp.choice('_1_02_wcfmp_1__04__07_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcfmp_1__04__08_S':hp.choice('_1_02_wcfmp_1__04__08_S', ['1', ]),
+ '_1_02_wcfmp_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wcfsgd_0_QUOTE_START_B':'weka.classifiers.functions.SGD',
+ '_1_02_wcfsgd_1__05__01_F':hp.choice('_1_02_wcfsgd_1__05__01_F', ['2', '1', '0', ]),
+ '_1_02_wcfsgd_1__05__02_L':hp.loguniform('_1_02_wcfsgd_1__05__02_L', math.log(1.0E-5), math.log(0.1)),
+ '_1_02_wcfsgd_1__05__03_R':hp.loguniform('_1_02_wcfsgd_1__05__03_R', math.log(1.0E-12), math.log(10.0)),
+ '_1_02_wcfsgd_1__05__04_N':hp.choice('_1_02_wcfsgd_1__05__04_N', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcfsgd_1__05__05_M':hp.choice('_1_02_wcfsgd_1__05__05_M', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcfsgd_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wcfsmo_0_QUOTE_START_B':'weka.classifiers.functions.SMO',
+ '_1_02_wcfsmo_1__06__01_0_C':hp.uniform('_1_02_wcfsmo_1__06__01_0_C', 0.5, 1.5),
+ '_1_02_wcfsmo_1__06__02_1_N':hp.choice('_1_02_wcfsmo_1__06__02_1_N', ['2', '1', '0', ]),
+ '_1_02_wcfsmo_1__06__03_2_M':hp.choice('_1_02_wcfsmo_1__06__03_2_M', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcfsmo_1__06__04_5_QUOTE_END':hp.choice('_1_02_wcfsmo_1__06__04_5_QUOTE_END', ['REMOVED', ]),
+ '_1_02_wcfsmo_1__06__05_auto_param_271':hp.choice('_1_02_wcfsmo_1__06__05_auto_param_271', [
+ {
+ '_1_02_wcfsmo_1__06__05__00__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.NormalizedPolyKernel',
+ '_1_02_wcfsmo_1__06__05__00__01_4_npoly_E':hp.uniform('_1_02_wcfsmo_1__06__05__00__01_4_npoly_E', 0.2, 5.0),
+ '_1_02_wcfsmo_1__06__05__00__02_4_npoly_L':hp.choice('_1_02_wcfsmo_1__06__05__00__02_4_npoly_L', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_02_wcfsmo_1__06__05__01__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.PolyKernel',
+ '_1_02_wcfsmo_1__06__05__01__01_4_poly_E':hp.uniform('_1_02_wcfsmo_1__06__05__01__01_4_poly_E', 0.2, 5.0),
+ '_1_02_wcfsmo_1__06__05__01__02_4_poly_L':hp.choice('_1_02_wcfsmo_1__06__05__01__02_4_poly_L', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_02_wcfsmo_1__06__05__02__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.Puk',
+ '_1_02_wcfsmo_1__06__05__02__01_4_puk_S':hp.uniform('_1_02_wcfsmo_1__06__05__02__01_4_puk_S', 0.1, 10.0),
+ '_1_02_wcfsmo_1__06__05__02__02_4_puk_O':hp.uniform('_1_02_wcfsmo_1__06__05__02__02_4_puk_O', 0.1, 1.0),
+ },
+ {
+ '_1_02_wcfsmo_1__06__05__03__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.RBFKernel',
+ '_1_02_wcfsmo_1__06__05__03__01_4_rbf_G':hp.loguniform('_1_02_wcfsmo_1__06__05__03__01_4_rbf_G', math.log(1.0E-4), math.log(1.0)),
+ },
+ ]),
+ '_1_02_wcfsmo_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wcfsl_0_QUOTE_START_B':'weka.classifiers.functions.SimpleLogistic',
+ '_1_02_wcfsl_1__07__01_S':hp.choice('_1_02_wcfsl_1__07__01_S', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcfsl_1__07__02_A':hp.choice('_1_02_wcfsl_1__07__02_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcfsl_1__07__03_auto_param_277':hp.choice('_1_02_wcfsl_1__07__03_auto_param_277', [
+ {
+ '_1_02_wcfsl_1__07__03__00__00_W_HIDDEN':'0',
+ '_1_02_wcfsl_1__07__03__00__01_1_W':hp.choice('_1_02_wcfsl_1__07__03__00__01_1_W', ['0', ]),
+ },
+ {
+ '_1_02_wcfsl_1__07__03__01__00_W_HIDDEN':'1',
+ '_1_02_wcfsl_1__07__03__01__01_2_W':hp.uniform('_1_02_wcfsl_1__07__03__01__01_2_W', 0.0, 1.0),
+ },
+ ]),
+ '_1_02_wcfsl_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wcfvp_0_QUOTE_START_B':'weka.classifiers.functions.VotedPerceptron',
+ '_1_02_wcfvp_1__08__01_INT_I':hp.quniform('_1_02_wcfvp_1__08__01_INT_I', 1.0, 10.0,1.0),
+ '_1_02_wcfvp_1__08__02_INT_M':hp.qloguniform('_1_02_wcfvp_1__08__02_INT_M', math.log(5000.0), math.log(50000.0), 1.0),
+ '_1_02_wcfvp_1__08__03_E':hp.uniform('_1_02_wcfvp_1__08__03_E', 0.2, 5.0),
+ '_1_02_wcfvp_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wclib_0_QUOTE_START_B':'weka.classifiers.lazy.IBk',
+ '_1_02_wclib_1__09__01_E':hp.choice('_1_02_wclib_1__09__01_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wclib_1__09__02_INT_K':hp.qloguniform('_1_02_wclib_1__09__02_INT_K', math.log(1.0), math.log(64.0), 1.0),
+ '_1_02_wclib_1__09__03_X':hp.choice('_1_02_wclib_1__09__03_X', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wclib_1__09__04_F':hp.choice('_1_02_wclib_1__09__04_F', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wclib_1__09__05_I':hp.choice('_1_02_wclib_1__09__05_I', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wclib_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wclks_0_QUOTE_START_B':'weka.classifiers.lazy.KStar',
+ '_1_02_wclks_1__10__01_INT_B':hp.quniform('_1_02_wclks_1__10__01_INT_B', 1.0, 100.0,1.0),
+ '_1_02_wclks_1__10__02_E':hp.choice('_1_02_wclks_1__10__02_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wclks_1__10__03_M':hp.choice('_1_02_wclks_1__10__03_M', ['n', 'd', 'm', 'a', ]),
+ '_1_02_wclks_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wcrdt_0_QUOTE_START_B':'weka.classifiers.rules.DecisionTable',
+ '_1_02_wcrdt_1__11__01_E':hp.choice('_1_02_wcrdt_1__11__01_E', ['auc', 'rmse', 'mae', 'acc', ]),
+ '_1_02_wcrdt_1__11__02_I':hp.choice('_1_02_wcrdt_1__11__02_I', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcrdt_1__11__03_S':hp.choice('_1_02_wcrdt_1__11__03_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]),
+ '_1_02_wcrdt_1__11__04_X':hp.choice('_1_02_wcrdt_1__11__04_X', ['4', '2', '3', '1', ]),
+ '_1_02_wcrdt_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wcrjr_0_QUOTE_START_B':'weka.classifiers.rules.JRip',
+ '_1_02_wcrjr_1__12__01_N':hp.uniform('_1_02_wcrjr_1__12__01_N', 1.0, 5.0),
+ '_1_02_wcrjr_1__12__02_E':hp.choice('_1_02_wcrjr_1__12__02_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcrjr_1__12__03_P':hp.choice('_1_02_wcrjr_1__12__03_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcrjr_1__12__04_INT_O':hp.quniform('_1_02_wcrjr_1__12__04_INT_O', 1.0, 5.0,1.0),
+ '_1_02_wcrjr_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wcror_0_QUOTE_START_B':'weka.classifiers.rules.OneR',
+ '_1_02_wcror_1__13__01_INT_B':hp.qloguniform('_1_02_wcror_1__13__01_INT_B', math.log(1.0), math.log(32.0), 1.0),
+ '_1_02_wcror_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wcrpart_0_QUOTE_START_B':'weka.classifiers.rules.PART',
+ '_1_02_wcrpart_1__14__01_INT_N':hp.quniform('_1_02_wcrpart_1__14__01_INT_N', 2.0, 5.0,1.0),
+ '_1_02_wcrpart_1__14__02_INT_M':hp.qloguniform('_1_02_wcrpart_1__14__02_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_02_wcrpart_1__14__03_R':hp.choice('_1_02_wcrpart_1__14__03_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcrpart_1__14__04_B':hp.choice('_1_02_wcrpart_1__14__04_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wcrpart_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wcrzr_0_QUOTE_START_B':'weka.classifiers.rules.ZeroR',
+ '_1_02_wcrzr_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wctds_0_QUOTE_START_B':'weka.classifiers.trees.DecisionStump',
+ '_1_02_wctds_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wctj_0_QUOTE_START_B':'weka.classifiers.trees.J48',
+ '_1_02_wctj_1__17__01_O':hp.choice('_1_02_wctj_1__17__01_O', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wctj_1__17__02_U':hp.choice('_1_02_wctj_1__17__02_U', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wctj_1__17__03_B':hp.choice('_1_02_wctj_1__17__03_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wctj_1__17__04_J':hp.choice('_1_02_wctj_1__17__04_J', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wctj_1__17__05_A':hp.choice('_1_02_wctj_1__17__05_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wctj_1__17__06_S':hp.choice('_1_02_wctj_1__17__06_S', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wctj_1__17__07_INT_M':hp.qloguniform('_1_02_wctj_1__17__07_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_02_wctj_1__17__08_C':hp.uniform('_1_02_wctj_1__17__08_C', 0.0, 1.0),
+ '_1_02_wctj_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wctlmt_0_QUOTE_START_B':'weka.classifiers.trees.LMT',
+ '_1_02_wctlmt_1__18__01_B':hp.choice('_1_02_wctlmt_1__18__01_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wctlmt_1__18__02_R':hp.choice('_1_02_wctlmt_1__18__02_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wctlmt_1__18__03_C':hp.choice('_1_02_wctlmt_1__18__03_C', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wctlmt_1__18__04_P':hp.choice('_1_02_wctlmt_1__18__04_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wctlmt_1__18__05_INT_M':hp.qloguniform('_1_02_wctlmt_1__18__05_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_02_wctlmt_1__18__06_A':hp.choice('_1_02_wctlmt_1__18__06_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wctlmt_1__18__07_auto_param_291':hp.choice('_1_02_wctlmt_1__18__07_auto_param_291', [
+ {
+ '_1_02_wctlmt_1__18__07__00__00_W_HIDDEN':'0',
+ '_1_02_wctlmt_1__18__07__00__01_1_W':hp.choice('_1_02_wctlmt_1__18__07__00__01_1_W', ['0', ]),
+ },
+ {
+ '_1_02_wctlmt_1__18__07__01__00_W_HIDDEN':'1',
+ '_1_02_wctlmt_1__18__07__01__01_2_W':hp.uniform('_1_02_wctlmt_1__18__07__01__01_2_W', 0.0, 1.0),
+ },
+ ]),
+ '_1_02_wctlmt_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wctrept_0_QUOTE_START_B':'weka.classifiers.trees.REPTree',
+ '_1_02_wctrept_1__19__01_INT_M':hp.qloguniform('_1_02_wctrept_1__19__01_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_02_wctrept_1__19__02_V':hp.loguniform('_1_02_wctrept_1__19__02_V', math.log(1.0E-5), math.log(0.1)),
+ '_1_02_wctrept_1__19__03_P':hp.choice('_1_02_wctrept_1__19__03_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wctrept_1__19__04_auto_param_295':hp.choice('_1_02_wctrept_1__19__04_auto_param_295', [
+ {
+ '_1_02_wctrept_1__19__04__00__00_depth_HIDDEN':'0',
+ '_1_02_wctrept_1__19__04__00__01_1_INT_L':hp.choice('_1_02_wctrept_1__19__04__00__01_1_INT_L', ['-1', ]),
+ },
+ {
+ '_1_02_wctrept_1__19__04__01__00_depth_HIDDEN':'1',
+ '_1_02_wctrept_1__19__04__01__01_2_INT_L':hp.quniform('_1_02_wctrept_1__19__04__01__01_2_INT_L', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_02_wctrept_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wctrf_0_QUOTE_START_B':'weka.classifiers.trees.RandomForest',
+ '_1_02_wctrf_1__20__01_INT_I':hp.qloguniform('_1_02_wctrf_1__20__01_INT_I', math.log(2.0), math.log(256.0), 1.0),
+ '_1_02_wctrf_1__20__02_auto_param_299':hp.choice('_1_02_wctrf_1__20__02_auto_param_299', [
+ {
+ '_1_02_wctrf_1__20__02__00__00_features_HIDDEN':'0',
+ '_1_02_wctrf_1__20__02__00__01_1_INT_K':hp.choice('_1_02_wctrf_1__20__02__00__01_1_INT_K', ['1', ]),
+ },
+ {
+ '_1_02_wctrf_1__20__02__01__00_features_HIDDEN':'1',
+ '_1_02_wctrf_1__20__02__01__01_2_INT_K':hp.qloguniform('_1_02_wctrf_1__20__02__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0),
+ },
+ ]),
+ '_1_02_wctrf_1__20__03_auto_param_302':hp.choice('_1_02_wctrf_1__20__03_auto_param_302', [
+ {
+ '_1_02_wctrf_1__20__03__00__00_depth_HIDDEN':'0',
+ '_1_02_wctrf_1__20__03__00__01_1_INT_depth':hp.choice('_1_02_wctrf_1__20__03__00__01_1_INT_depth', ['1', ]),
+ },
+ {
+ '_1_02_wctrf_1__20__03__01__00_depth_HIDDEN':'1',
+ '_1_02_wctrf_1__20__03__01__01_2_INT_depth':hp.quniform('_1_02_wctrf_1__20__03__01__01_2_INT_depth', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_02_wctrf_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_02_wctrt_0_QUOTE_START_B':'weka.classifiers.trees.RandomTree',
+ '_1_02_wctrt_1__21__01_INT_M':hp.qloguniform('_1_02_wctrt_1__21__01_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_02_wctrt_1__21__02_U':hp.choice('_1_02_wctrt_1__21__02_U', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_02_wctrt_1__21__03_auto_param_306':hp.choice('_1_02_wctrt_1__21__03_auto_param_306', [
+ {
+ '_1_02_wctrt_1__21__03__00__00_features_HIDDEN':'0',
+ '_1_02_wctrt_1__21__03__00__01_1_INT_K':hp.choice('_1_02_wctrt_1__21__03__00__01_1_INT_K', ['0', ]),
+ },
+ {
+ '_1_02_wctrt_1__21__03__01__00_features_HIDDEN':'1',
+ '_1_02_wctrt_1__21__03__01__01_2_INT_K':hp.qloguniform('_1_02_wctrt_1__21__03__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0),
+ },
+ ]),
+ '_1_02_wctrt_1__21__04_auto_param_309':hp.choice('_1_02_wctrt_1__21__04_auto_param_309', [
+ {
+ '_1_02_wctrt_1__21__04__00__00_depth_HIDDEN':'0',
+ '_1_02_wctrt_1__21__04__00__01_1_INT_depth':hp.choice('_1_02_wctrt_1__21__04__00__01_1_INT_depth', ['0', ]),
+ },
+ {
+ '_1_02_wctrt_1__21__04__01__00_depth_HIDDEN':'1',
+ '_1_02_wctrt_1__21__04__01__01_2_INT_depth':hp.quniform('_1_02_wctrt_1__21__04__01__01_2_INT_depth', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_02_wctrt_1__21__05_auto_param_312':hp.choice('_1_02_wctrt_1__21__05_auto_param_312', [
+ {
+ '_1_02_wctrt_1__21__05__00__00_back_HIDDEN':'0',
+ '_1_02_wctrt_1__21__05__00__01_1_INT_N':hp.choice('_1_02_wctrt_1__21__05__00__01_1_INT_N', ['0', ]),
+ },
+ {
+ '_1_02_wctrt_1__21__05__01__00_back_HIDDEN':'1',
+ '_1_02_wctrt_1__21__05__01__01_2_INT_N':hp.quniform('_1_02_wctrt_1__21__05__01__01_2_INT_N', 2.0, 5.0,1.0),
+ },
+ ]),
+ '_1_02_wctrt_2_QUOTE_END':'REMOVED',
+ },
+ ]),
+ hp.choice('auto_param_315', [
+ (
+ ),
+ (
+ hp.choice('auto_param_318', [
+ {
+ '_1_03_wcbbn_0_QUOTE_START_B':'weka.classifiers.bayes.BayesNet',
+ '_1_03_wcbbn_1__00__01_D':hp.choice('_1_03_wcbbn_1__00__01_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcbbn_1__00__02_Q':hp.choice('_1_03_wcbbn_1__00__02_Q', ['weka.classifiers.bayes.net.search.local.TAN', 'weka.classifiers.bayes.net.search.local.HillClimber', 'weka.classifiers.bayes.net.search.local.LAGDHillClimber', 'weka.classifiers.bayes.net.search.local.SimulatedAnnealing', 'weka.classifiers.bayes.net.search.local.TabuSearch', 'weka.classifiers.bayes.net.search.local.K2', ]),
+ '_1_03_wcbbn_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wcbnb_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayes',
+ '_1_03_wcbnb_1__01__01_K':hp.choice('_1_03_wcbnb_1__01__01_K', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcbnb_1__01__02_D':hp.choice('_1_03_wcbnb_1__01__02_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcbnb_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wcbnbm_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayesMultinomial',
+ '_1_03_wcbnbm_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wcfl_0_QUOTE_START_B':'weka.classifiers.functions.Logistic',
+ '_1_03_wcfl_1__03__01_R':hp.loguniform('_1_03_wcfl_1__03__01_R', math.log(1.0E-12), math.log(10.0)),
+ '_1_03_wcfl_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wcfmp_0_QUOTE_START_B':'weka.classifiers.functions.MultilayerPerceptron',
+ '_1_03_wcfmp_1__04__01_L':hp.uniform('_1_03_wcfmp_1__04__01_L', 0.1, 1.0),
+ '_1_03_wcfmp_1__04__02_M':hp.uniform('_1_03_wcfmp_1__04__02_M', 0.1, 1.0),
+ '_1_03_wcfmp_1__04__03_B':hp.choice('_1_03_wcfmp_1__04__03_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcfmp_1__04__04_H':hp.choice('_1_03_wcfmp_1__04__04_H', ['t', 'i', 'o', 'a', ]),
+ '_1_03_wcfmp_1__04__05_C':hp.choice('_1_03_wcfmp_1__04__05_C', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcfmp_1__04__06_R':hp.choice('_1_03_wcfmp_1__04__06_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcfmp_1__04__07_D':hp.choice('_1_03_wcfmp_1__04__07_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcfmp_1__04__08_S':hp.choice('_1_03_wcfmp_1__04__08_S', ['1', ]),
+ '_1_03_wcfmp_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wcfsgd_0_QUOTE_START_B':'weka.classifiers.functions.SGD',
+ '_1_03_wcfsgd_1__05__01_F':hp.choice('_1_03_wcfsgd_1__05__01_F', ['2', '1', '0', ]),
+ '_1_03_wcfsgd_1__05__02_L':hp.loguniform('_1_03_wcfsgd_1__05__02_L', math.log(1.0E-5), math.log(0.1)),
+ '_1_03_wcfsgd_1__05__03_R':hp.loguniform('_1_03_wcfsgd_1__05__03_R', math.log(1.0E-12), math.log(10.0)),
+ '_1_03_wcfsgd_1__05__04_N':hp.choice('_1_03_wcfsgd_1__05__04_N', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcfsgd_1__05__05_M':hp.choice('_1_03_wcfsgd_1__05__05_M', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcfsgd_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wcfsmo_0_QUOTE_START_B':'weka.classifiers.functions.SMO',
+ '_1_03_wcfsmo_1__06__01_0_C':hp.uniform('_1_03_wcfsmo_1__06__01_0_C', 0.5, 1.5),
+ '_1_03_wcfsmo_1__06__02_1_N':hp.choice('_1_03_wcfsmo_1__06__02_1_N', ['2', '1', '0', ]),
+ '_1_03_wcfsmo_1__06__03_2_M':hp.choice('_1_03_wcfsmo_1__06__03_2_M', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcfsmo_1__06__04_5_QUOTE_END':hp.choice('_1_03_wcfsmo_1__06__04_5_QUOTE_END', ['REMOVED', ]),
+ '_1_03_wcfsmo_1__06__05_auto_param_326':hp.choice('_1_03_wcfsmo_1__06__05_auto_param_326', [
+ {
+ '_1_03_wcfsmo_1__06__05__00__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.NormalizedPolyKernel',
+ '_1_03_wcfsmo_1__06__05__00__01_4_npoly_E':hp.uniform('_1_03_wcfsmo_1__06__05__00__01_4_npoly_E', 0.2, 5.0),
+ '_1_03_wcfsmo_1__06__05__00__02_4_npoly_L':hp.choice('_1_03_wcfsmo_1__06__05__00__02_4_npoly_L', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_03_wcfsmo_1__06__05__01__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.PolyKernel',
+ '_1_03_wcfsmo_1__06__05__01__01_4_poly_E':hp.uniform('_1_03_wcfsmo_1__06__05__01__01_4_poly_E', 0.2, 5.0),
+ '_1_03_wcfsmo_1__06__05__01__02_4_poly_L':hp.choice('_1_03_wcfsmo_1__06__05__01__02_4_poly_L', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_03_wcfsmo_1__06__05__02__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.Puk',
+ '_1_03_wcfsmo_1__06__05__02__01_4_puk_S':hp.uniform('_1_03_wcfsmo_1__06__05__02__01_4_puk_S', 0.1, 10.0),
+ '_1_03_wcfsmo_1__06__05__02__02_4_puk_O':hp.uniform('_1_03_wcfsmo_1__06__05__02__02_4_puk_O', 0.1, 1.0),
+ },
+ {
+ '_1_03_wcfsmo_1__06__05__03__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.RBFKernel',
+ '_1_03_wcfsmo_1__06__05__03__01_4_rbf_G':hp.loguniform('_1_03_wcfsmo_1__06__05__03__01_4_rbf_G', math.log(1.0E-4), math.log(1.0)),
+ },
+ ]),
+ '_1_03_wcfsmo_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wcfsl_0_QUOTE_START_B':'weka.classifiers.functions.SimpleLogistic',
+ '_1_03_wcfsl_1__07__01_S':hp.choice('_1_03_wcfsl_1__07__01_S', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcfsl_1__07__02_A':hp.choice('_1_03_wcfsl_1__07__02_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcfsl_1__07__03_auto_param_332':hp.choice('_1_03_wcfsl_1__07__03_auto_param_332', [
+ {
+ '_1_03_wcfsl_1__07__03__00__00_W_HIDDEN':'0',
+ '_1_03_wcfsl_1__07__03__00__01_1_W':hp.choice('_1_03_wcfsl_1__07__03__00__01_1_W', ['0', ]),
+ },
+ {
+ '_1_03_wcfsl_1__07__03__01__00_W_HIDDEN':'1',
+ '_1_03_wcfsl_1__07__03__01__01_2_W':hp.uniform('_1_03_wcfsl_1__07__03__01__01_2_W', 0.0, 1.0),
+ },
+ ]),
+ '_1_03_wcfsl_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wcfvp_0_QUOTE_START_B':'weka.classifiers.functions.VotedPerceptron',
+ '_1_03_wcfvp_1__08__01_INT_I':hp.quniform('_1_03_wcfvp_1__08__01_INT_I', 1.0, 10.0,1.0),
+ '_1_03_wcfvp_1__08__02_INT_M':hp.qloguniform('_1_03_wcfvp_1__08__02_INT_M', math.log(5000.0), math.log(50000.0), 1.0),
+ '_1_03_wcfvp_1__08__03_E':hp.uniform('_1_03_wcfvp_1__08__03_E', 0.2, 5.0),
+ '_1_03_wcfvp_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wclib_0_QUOTE_START_B':'weka.classifiers.lazy.IBk',
+ '_1_03_wclib_1__09__01_E':hp.choice('_1_03_wclib_1__09__01_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wclib_1__09__02_INT_K':hp.qloguniform('_1_03_wclib_1__09__02_INT_K', math.log(1.0), math.log(64.0), 1.0),
+ '_1_03_wclib_1__09__03_X':hp.choice('_1_03_wclib_1__09__03_X', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wclib_1__09__04_F':hp.choice('_1_03_wclib_1__09__04_F', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wclib_1__09__05_I':hp.choice('_1_03_wclib_1__09__05_I', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wclib_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wclks_0_QUOTE_START_B':'weka.classifiers.lazy.KStar',
+ '_1_03_wclks_1__10__01_INT_B':hp.quniform('_1_03_wclks_1__10__01_INT_B', 1.0, 100.0,1.0),
+ '_1_03_wclks_1__10__02_E':hp.choice('_1_03_wclks_1__10__02_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wclks_1__10__03_M':hp.choice('_1_03_wclks_1__10__03_M', ['n', 'd', 'm', 'a', ]),
+ '_1_03_wclks_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wcrdt_0_QUOTE_START_B':'weka.classifiers.rules.DecisionTable',
+ '_1_03_wcrdt_1__11__01_E':hp.choice('_1_03_wcrdt_1__11__01_E', ['auc', 'rmse', 'mae', 'acc', ]),
+ '_1_03_wcrdt_1__11__02_I':hp.choice('_1_03_wcrdt_1__11__02_I', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcrdt_1__11__03_S':hp.choice('_1_03_wcrdt_1__11__03_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]),
+ '_1_03_wcrdt_1__11__04_X':hp.choice('_1_03_wcrdt_1__11__04_X', ['4', '2', '3', '1', ]),
+ '_1_03_wcrdt_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wcrjr_0_QUOTE_START_B':'weka.classifiers.rules.JRip',
+ '_1_03_wcrjr_1__12__01_N':hp.uniform('_1_03_wcrjr_1__12__01_N', 1.0, 5.0),
+ '_1_03_wcrjr_1__12__02_E':hp.choice('_1_03_wcrjr_1__12__02_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcrjr_1__12__03_P':hp.choice('_1_03_wcrjr_1__12__03_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcrjr_1__12__04_INT_O':hp.quniform('_1_03_wcrjr_1__12__04_INT_O', 1.0, 5.0,1.0),
+ '_1_03_wcrjr_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wcror_0_QUOTE_START_B':'weka.classifiers.rules.OneR',
+ '_1_03_wcror_1__13__01_INT_B':hp.qloguniform('_1_03_wcror_1__13__01_INT_B', math.log(1.0), math.log(32.0), 1.0),
+ '_1_03_wcror_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wcrpart_0_QUOTE_START_B':'weka.classifiers.rules.PART',
+ '_1_03_wcrpart_1__14__01_INT_N':hp.quniform('_1_03_wcrpart_1__14__01_INT_N', 2.0, 5.0,1.0),
+ '_1_03_wcrpart_1__14__02_INT_M':hp.qloguniform('_1_03_wcrpart_1__14__02_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_03_wcrpart_1__14__03_R':hp.choice('_1_03_wcrpart_1__14__03_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcrpart_1__14__04_B':hp.choice('_1_03_wcrpart_1__14__04_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wcrpart_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wcrzr_0_QUOTE_START_B':'weka.classifiers.rules.ZeroR',
+ '_1_03_wcrzr_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wctds_0_QUOTE_START_B':'weka.classifiers.trees.DecisionStump',
+ '_1_03_wctds_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wctj_0_QUOTE_START_B':'weka.classifiers.trees.J48',
+ '_1_03_wctj_1__17__01_O':hp.choice('_1_03_wctj_1__17__01_O', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wctj_1__17__02_U':hp.choice('_1_03_wctj_1__17__02_U', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wctj_1__17__03_B':hp.choice('_1_03_wctj_1__17__03_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wctj_1__17__04_J':hp.choice('_1_03_wctj_1__17__04_J', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wctj_1__17__05_A':hp.choice('_1_03_wctj_1__17__05_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wctj_1__17__06_S':hp.choice('_1_03_wctj_1__17__06_S', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wctj_1__17__07_INT_M':hp.qloguniform('_1_03_wctj_1__17__07_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_03_wctj_1__17__08_C':hp.uniform('_1_03_wctj_1__17__08_C', 0.0, 1.0),
+ '_1_03_wctj_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wctlmt_0_QUOTE_START_B':'weka.classifiers.trees.LMT',
+ '_1_03_wctlmt_1__18__01_B':hp.choice('_1_03_wctlmt_1__18__01_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wctlmt_1__18__02_R':hp.choice('_1_03_wctlmt_1__18__02_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wctlmt_1__18__03_C':hp.choice('_1_03_wctlmt_1__18__03_C', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wctlmt_1__18__04_P':hp.choice('_1_03_wctlmt_1__18__04_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wctlmt_1__18__05_INT_M':hp.qloguniform('_1_03_wctlmt_1__18__05_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_03_wctlmt_1__18__06_A':hp.choice('_1_03_wctlmt_1__18__06_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wctlmt_1__18__07_auto_param_346':hp.choice('_1_03_wctlmt_1__18__07_auto_param_346', [
+ {
+ '_1_03_wctlmt_1__18__07__00__00_W_HIDDEN':'0',
+ '_1_03_wctlmt_1__18__07__00__01_1_W':hp.choice('_1_03_wctlmt_1__18__07__00__01_1_W', ['0', ]),
+ },
+ {
+ '_1_03_wctlmt_1__18__07__01__00_W_HIDDEN':'1',
+ '_1_03_wctlmt_1__18__07__01__01_2_W':hp.uniform('_1_03_wctlmt_1__18__07__01__01_2_W', 0.0, 1.0),
+ },
+ ]),
+ '_1_03_wctlmt_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wctrept_0_QUOTE_START_B':'weka.classifiers.trees.REPTree',
+ '_1_03_wctrept_1__19__01_INT_M':hp.qloguniform('_1_03_wctrept_1__19__01_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_03_wctrept_1__19__02_V':hp.loguniform('_1_03_wctrept_1__19__02_V', math.log(1.0E-5), math.log(0.1)),
+ '_1_03_wctrept_1__19__03_P':hp.choice('_1_03_wctrept_1__19__03_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wctrept_1__19__04_auto_param_350':hp.choice('_1_03_wctrept_1__19__04_auto_param_350', [
+ {
+ '_1_03_wctrept_1__19__04__00__00_depth_HIDDEN':'0',
+ '_1_03_wctrept_1__19__04__00__01_1_INT_L':hp.choice('_1_03_wctrept_1__19__04__00__01_1_INT_L', ['-1', ]),
+ },
+ {
+ '_1_03_wctrept_1__19__04__01__00_depth_HIDDEN':'1',
+ '_1_03_wctrept_1__19__04__01__01_2_INT_L':hp.quniform('_1_03_wctrept_1__19__04__01__01_2_INT_L', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_03_wctrept_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wctrf_0_QUOTE_START_B':'weka.classifiers.trees.RandomForest',
+ '_1_03_wctrf_1__20__01_INT_I':hp.qloguniform('_1_03_wctrf_1__20__01_INT_I', math.log(2.0), math.log(256.0), 1.0),
+ '_1_03_wctrf_1__20__02_auto_param_354':hp.choice('_1_03_wctrf_1__20__02_auto_param_354', [
+ {
+ '_1_03_wctrf_1__20__02__00__00_features_HIDDEN':'0',
+ '_1_03_wctrf_1__20__02__00__01_1_INT_K':hp.choice('_1_03_wctrf_1__20__02__00__01_1_INT_K', ['1', ]),
+ },
+ {
+ '_1_03_wctrf_1__20__02__01__00_features_HIDDEN':'1',
+ '_1_03_wctrf_1__20__02__01__01_2_INT_K':hp.qloguniform('_1_03_wctrf_1__20__02__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0),
+ },
+ ]),
+ '_1_03_wctrf_1__20__03_auto_param_357':hp.choice('_1_03_wctrf_1__20__03_auto_param_357', [
+ {
+ '_1_03_wctrf_1__20__03__00__00_depth_HIDDEN':'0',
+ '_1_03_wctrf_1__20__03__00__01_1_INT_depth':hp.choice('_1_03_wctrf_1__20__03__00__01_1_INT_depth', ['1', ]),
+ },
+ {
+ '_1_03_wctrf_1__20__03__01__00_depth_HIDDEN':'1',
+ '_1_03_wctrf_1__20__03__01__01_2_INT_depth':hp.quniform('_1_03_wctrf_1__20__03__01__01_2_INT_depth', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_03_wctrf_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_03_wctrt_0_QUOTE_START_B':'weka.classifiers.trees.RandomTree',
+ '_1_03_wctrt_1__21__01_INT_M':hp.qloguniform('_1_03_wctrt_1__21__01_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_03_wctrt_1__21__02_U':hp.choice('_1_03_wctrt_1__21__02_U', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_03_wctrt_1__21__03_auto_param_361':hp.choice('_1_03_wctrt_1__21__03_auto_param_361', [
+ {
+ '_1_03_wctrt_1__21__03__00__00_features_HIDDEN':'0',
+ '_1_03_wctrt_1__21__03__00__01_1_INT_K':hp.choice('_1_03_wctrt_1__21__03__00__01_1_INT_K', ['0', ]),
+ },
+ {
+ '_1_03_wctrt_1__21__03__01__00_features_HIDDEN':'1',
+ '_1_03_wctrt_1__21__03__01__01_2_INT_K':hp.qloguniform('_1_03_wctrt_1__21__03__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0),
+ },
+ ]),
+ '_1_03_wctrt_1__21__04_auto_param_364':hp.choice('_1_03_wctrt_1__21__04_auto_param_364', [
+ {
+ '_1_03_wctrt_1__21__04__00__00_depth_HIDDEN':'0',
+ '_1_03_wctrt_1__21__04__00__01_1_INT_depth':hp.choice('_1_03_wctrt_1__21__04__00__01_1_INT_depth', ['0', ]),
+ },
+ {
+ '_1_03_wctrt_1__21__04__01__00_depth_HIDDEN':'1',
+ '_1_03_wctrt_1__21__04__01__01_2_INT_depth':hp.quniform('_1_03_wctrt_1__21__04__01__01_2_INT_depth', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_03_wctrt_1__21__05_auto_param_367':hp.choice('_1_03_wctrt_1__21__05_auto_param_367', [
+ {
+ '_1_03_wctrt_1__21__05__00__00_back_HIDDEN':'0',
+ '_1_03_wctrt_1__21__05__00__01_1_INT_N':hp.choice('_1_03_wctrt_1__21__05__00__01_1_INT_N', ['0', ]),
+ },
+ {
+ '_1_03_wctrt_1__21__05__01__00_back_HIDDEN':'1',
+ '_1_03_wctrt_1__21__05__01__01_2_INT_N':hp.quniform('_1_03_wctrt_1__21__05__01__01_2_INT_N', 2.0, 5.0,1.0),
+ },
+ ]),
+ '_1_03_wctrt_2_QUOTE_END':'REMOVED',
+ },
+ ]),
+ hp.choice('auto_param_370', [
+ (
+ ),
+ (
+ hp.choice('auto_param_373', [
+ {
+ '_1_04_wcbbn_0_QUOTE_START_B':'weka.classifiers.bayes.BayesNet',
+ '_1_04_wcbbn_1__00__01_D':hp.choice('_1_04_wcbbn_1__00__01_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcbbn_1__00__02_Q':hp.choice('_1_04_wcbbn_1__00__02_Q', ['weka.classifiers.bayes.net.search.local.TAN', 'weka.classifiers.bayes.net.search.local.HillClimber', 'weka.classifiers.bayes.net.search.local.LAGDHillClimber', 'weka.classifiers.bayes.net.search.local.SimulatedAnnealing', 'weka.classifiers.bayes.net.search.local.TabuSearch', 'weka.classifiers.bayes.net.search.local.K2', ]),
+ '_1_04_wcbbn_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wcbnb_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayes',
+ '_1_04_wcbnb_1__01__01_K':hp.choice('_1_04_wcbnb_1__01__01_K', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcbnb_1__01__02_D':hp.choice('_1_04_wcbnb_1__01__02_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcbnb_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wcbnbm_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayesMultinomial',
+ '_1_04_wcbnbm_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wcfl_0_QUOTE_START_B':'weka.classifiers.functions.Logistic',
+ '_1_04_wcfl_1__03__01_R':hp.loguniform('_1_04_wcfl_1__03__01_R', math.log(1.0E-12), math.log(10.0)),
+ '_1_04_wcfl_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wcfmp_0_QUOTE_START_B':'weka.classifiers.functions.MultilayerPerceptron',
+ '_1_04_wcfmp_1__04__01_L':hp.uniform('_1_04_wcfmp_1__04__01_L', 0.1, 1.0),
+ '_1_04_wcfmp_1__04__02_M':hp.uniform('_1_04_wcfmp_1__04__02_M', 0.1, 1.0),
+ '_1_04_wcfmp_1__04__03_B':hp.choice('_1_04_wcfmp_1__04__03_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcfmp_1__04__04_H':hp.choice('_1_04_wcfmp_1__04__04_H', ['t', 'i', 'o', 'a', ]),
+ '_1_04_wcfmp_1__04__05_C':hp.choice('_1_04_wcfmp_1__04__05_C', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcfmp_1__04__06_R':hp.choice('_1_04_wcfmp_1__04__06_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcfmp_1__04__07_D':hp.choice('_1_04_wcfmp_1__04__07_D', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcfmp_1__04__08_S':hp.choice('_1_04_wcfmp_1__04__08_S', ['1', ]),
+ '_1_04_wcfmp_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wcfsgd_0_QUOTE_START_B':'weka.classifiers.functions.SGD',
+ '_1_04_wcfsgd_1__05__01_F':hp.choice('_1_04_wcfsgd_1__05__01_F', ['2', '1', '0', ]),
+ '_1_04_wcfsgd_1__05__02_L':hp.loguniform('_1_04_wcfsgd_1__05__02_L', math.log(1.0E-5), math.log(0.1)),
+ '_1_04_wcfsgd_1__05__03_R':hp.loguniform('_1_04_wcfsgd_1__05__03_R', math.log(1.0E-12), math.log(10.0)),
+ '_1_04_wcfsgd_1__05__04_N':hp.choice('_1_04_wcfsgd_1__05__04_N', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcfsgd_1__05__05_M':hp.choice('_1_04_wcfsgd_1__05__05_M', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcfsgd_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wcfsmo_0_QUOTE_START_B':'weka.classifiers.functions.SMO',
+ '_1_04_wcfsmo_1__06__01_0_C':hp.uniform('_1_04_wcfsmo_1__06__01_0_C', 0.5, 1.5),
+ '_1_04_wcfsmo_1__06__02_1_N':hp.choice('_1_04_wcfsmo_1__06__02_1_N', ['2', '1', '0', ]),
+ '_1_04_wcfsmo_1__06__03_2_M':hp.choice('_1_04_wcfsmo_1__06__03_2_M', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcfsmo_1__06__04_5_QUOTE_END':hp.choice('_1_04_wcfsmo_1__06__04_5_QUOTE_END', ['REMOVED', ]),
+ '_1_04_wcfsmo_1__06__05_auto_param_381':hp.choice('_1_04_wcfsmo_1__06__05_auto_param_381', [
+ {
+ '_1_04_wcfsmo_1__06__05__00__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.NormalizedPolyKernel',
+ '_1_04_wcfsmo_1__06__05__00__01_4_npoly_E':hp.uniform('_1_04_wcfsmo_1__06__05__00__01_4_npoly_E', 0.2, 5.0),
+ '_1_04_wcfsmo_1__06__05__00__02_4_npoly_L':hp.choice('_1_04_wcfsmo_1__06__05__00__02_4_npoly_L', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_04_wcfsmo_1__06__05__01__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.PolyKernel',
+ '_1_04_wcfsmo_1__06__05__01__01_4_poly_E':hp.uniform('_1_04_wcfsmo_1__06__05__01__01_4_poly_E', 0.2, 5.0),
+ '_1_04_wcfsmo_1__06__05__01__02_4_poly_L':hp.choice('_1_04_wcfsmo_1__06__05__01__02_4_poly_L', ['REMOVE_PREV', 'REMOVED', ]),
+ },
+ {
+ '_1_04_wcfsmo_1__06__05__02__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.Puk',
+ '_1_04_wcfsmo_1__06__05__02__01_4_puk_S':hp.uniform('_1_04_wcfsmo_1__06__05__02__01_4_puk_S', 0.1, 10.0),
+ '_1_04_wcfsmo_1__06__05__02__02_4_puk_O':hp.uniform('_1_04_wcfsmo_1__06__05__02__02_4_puk_O', 0.1, 1.0),
+ },
+ {
+ '_1_04_wcfsmo_1__06__05__03__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.RBFKernel',
+ '_1_04_wcfsmo_1__06__05__03__01_4_rbf_G':hp.loguniform('_1_04_wcfsmo_1__06__05__03__01_4_rbf_G', math.log(1.0E-4), math.log(1.0)),
+ },
+ ]),
+ '_1_04_wcfsmo_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wcfsl_0_QUOTE_START_B':'weka.classifiers.functions.SimpleLogistic',
+ '_1_04_wcfsl_1__07__01_S':hp.choice('_1_04_wcfsl_1__07__01_S', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcfsl_1__07__02_A':hp.choice('_1_04_wcfsl_1__07__02_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcfsl_1__07__03_auto_param_387':hp.choice('_1_04_wcfsl_1__07__03_auto_param_387', [
+ {
+ '_1_04_wcfsl_1__07__03__00__00_W_HIDDEN':'0',
+ '_1_04_wcfsl_1__07__03__00__01_1_W':hp.choice('_1_04_wcfsl_1__07__03__00__01_1_W', ['0', ]),
+ },
+ {
+ '_1_04_wcfsl_1__07__03__01__00_W_HIDDEN':'1',
+ '_1_04_wcfsl_1__07__03__01__01_2_W':hp.uniform('_1_04_wcfsl_1__07__03__01__01_2_W', 0.0, 1.0),
+ },
+ ]),
+ '_1_04_wcfsl_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wcfvp_0_QUOTE_START_B':'weka.classifiers.functions.VotedPerceptron',
+ '_1_04_wcfvp_1__08__01_INT_I':hp.quniform('_1_04_wcfvp_1__08__01_INT_I', 1.0, 10.0,1.0),
+ '_1_04_wcfvp_1__08__02_INT_M':hp.qloguniform('_1_04_wcfvp_1__08__02_INT_M', math.log(5000.0), math.log(50000.0), 1.0),
+ '_1_04_wcfvp_1__08__03_E':hp.uniform('_1_04_wcfvp_1__08__03_E', 0.2, 5.0),
+ '_1_04_wcfvp_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wclib_0_QUOTE_START_B':'weka.classifiers.lazy.IBk',
+ '_1_04_wclib_1__09__01_E':hp.choice('_1_04_wclib_1__09__01_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wclib_1__09__02_INT_K':hp.qloguniform('_1_04_wclib_1__09__02_INT_K', math.log(1.0), math.log(64.0), 1.0),
+ '_1_04_wclib_1__09__03_X':hp.choice('_1_04_wclib_1__09__03_X', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wclib_1__09__04_F':hp.choice('_1_04_wclib_1__09__04_F', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wclib_1__09__05_I':hp.choice('_1_04_wclib_1__09__05_I', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wclib_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wclks_0_QUOTE_START_B':'weka.classifiers.lazy.KStar',
+ '_1_04_wclks_1__10__01_INT_B':hp.quniform('_1_04_wclks_1__10__01_INT_B', 1.0, 100.0,1.0),
+ '_1_04_wclks_1__10__02_E':hp.choice('_1_04_wclks_1__10__02_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wclks_1__10__03_M':hp.choice('_1_04_wclks_1__10__03_M', ['n', 'd', 'm', 'a', ]),
+ '_1_04_wclks_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wcrdt_0_QUOTE_START_B':'weka.classifiers.rules.DecisionTable',
+ '_1_04_wcrdt_1__11__01_E':hp.choice('_1_04_wcrdt_1__11__01_E', ['auc', 'rmse', 'mae', 'acc', ]),
+ '_1_04_wcrdt_1__11__02_I':hp.choice('_1_04_wcrdt_1__11__02_I', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcrdt_1__11__03_S':hp.choice('_1_04_wcrdt_1__11__03_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]),
+ '_1_04_wcrdt_1__11__04_X':hp.choice('_1_04_wcrdt_1__11__04_X', ['4', '2', '3', '1', ]),
+ '_1_04_wcrdt_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wcrjr_0_QUOTE_START_B':'weka.classifiers.rules.JRip',
+ '_1_04_wcrjr_1__12__01_N':hp.uniform('_1_04_wcrjr_1__12__01_N', 1.0, 5.0),
+ '_1_04_wcrjr_1__12__02_E':hp.choice('_1_04_wcrjr_1__12__02_E', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcrjr_1__12__03_P':hp.choice('_1_04_wcrjr_1__12__03_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcrjr_1__12__04_INT_O':hp.quniform('_1_04_wcrjr_1__12__04_INT_O', 1.0, 5.0,1.0),
+ '_1_04_wcrjr_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wcror_0_QUOTE_START_B':'weka.classifiers.rules.OneR',
+ '_1_04_wcror_1__13__01_INT_B':hp.qloguniform('_1_04_wcror_1__13__01_INT_B', math.log(1.0), math.log(32.0), 1.0),
+ '_1_04_wcror_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wcrpart_0_QUOTE_START_B':'weka.classifiers.rules.PART',
+ '_1_04_wcrpart_1__14__01_INT_N':hp.quniform('_1_04_wcrpart_1__14__01_INT_N', 2.0, 5.0,1.0),
+ '_1_04_wcrpart_1__14__02_INT_M':hp.qloguniform('_1_04_wcrpart_1__14__02_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_04_wcrpart_1__14__03_R':hp.choice('_1_04_wcrpart_1__14__03_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcrpart_1__14__04_B':hp.choice('_1_04_wcrpart_1__14__04_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wcrpart_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wcrzr_0_QUOTE_START_B':'weka.classifiers.rules.ZeroR',
+ '_1_04_wcrzr_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wctds_0_QUOTE_START_B':'weka.classifiers.trees.DecisionStump',
+ '_1_04_wctds_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wctj_0_QUOTE_START_B':'weka.classifiers.trees.J48',
+ '_1_04_wctj_1__17__01_O':hp.choice('_1_04_wctj_1__17__01_O', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wctj_1__17__02_U':hp.choice('_1_04_wctj_1__17__02_U', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wctj_1__17__03_B':hp.choice('_1_04_wctj_1__17__03_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wctj_1__17__04_J':hp.choice('_1_04_wctj_1__17__04_J', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wctj_1__17__05_A':hp.choice('_1_04_wctj_1__17__05_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wctj_1__17__06_S':hp.choice('_1_04_wctj_1__17__06_S', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wctj_1__17__07_INT_M':hp.qloguniform('_1_04_wctj_1__17__07_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_04_wctj_1__17__08_C':hp.uniform('_1_04_wctj_1__17__08_C', 0.0, 1.0),
+ '_1_04_wctj_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wctlmt_0_QUOTE_START_B':'weka.classifiers.trees.LMT',
+ '_1_04_wctlmt_1__18__01_B':hp.choice('_1_04_wctlmt_1__18__01_B', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wctlmt_1__18__02_R':hp.choice('_1_04_wctlmt_1__18__02_R', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wctlmt_1__18__03_C':hp.choice('_1_04_wctlmt_1__18__03_C', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wctlmt_1__18__04_P':hp.choice('_1_04_wctlmt_1__18__04_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wctlmt_1__18__05_INT_M':hp.qloguniform('_1_04_wctlmt_1__18__05_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_04_wctlmt_1__18__06_A':hp.choice('_1_04_wctlmt_1__18__06_A', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wctlmt_1__18__07_auto_param_401':hp.choice('_1_04_wctlmt_1__18__07_auto_param_401', [
+ {
+ '_1_04_wctlmt_1__18__07__00__00_W_HIDDEN':'0',
+ '_1_04_wctlmt_1__18__07__00__01_1_W':hp.choice('_1_04_wctlmt_1__18__07__00__01_1_W', ['0', ]),
+ },
+ {
+ '_1_04_wctlmt_1__18__07__01__00_W_HIDDEN':'1',
+ '_1_04_wctlmt_1__18__07__01__01_2_W':hp.uniform('_1_04_wctlmt_1__18__07__01__01_2_W', 0.0, 1.0),
+ },
+ ]),
+ '_1_04_wctlmt_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wctrept_0_QUOTE_START_B':'weka.classifiers.trees.REPTree',
+ '_1_04_wctrept_1__19__01_INT_M':hp.qloguniform('_1_04_wctrept_1__19__01_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_04_wctrept_1__19__02_V':hp.loguniform('_1_04_wctrept_1__19__02_V', math.log(1.0E-5), math.log(0.1)),
+ '_1_04_wctrept_1__19__03_P':hp.choice('_1_04_wctrept_1__19__03_P', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wctrept_1__19__04_auto_param_405':hp.choice('_1_04_wctrept_1__19__04_auto_param_405', [
+ {
+ '_1_04_wctrept_1__19__04__00__00_depth_HIDDEN':'0',
+ '_1_04_wctrept_1__19__04__00__01_1_INT_L':hp.choice('_1_04_wctrept_1__19__04__00__01_1_INT_L', ['-1', ]),
+ },
+ {
+ '_1_04_wctrept_1__19__04__01__00_depth_HIDDEN':'1',
+ '_1_04_wctrept_1__19__04__01__01_2_INT_L':hp.quniform('_1_04_wctrept_1__19__04__01__01_2_INT_L', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_04_wctrept_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wctrf_0_QUOTE_START_B':'weka.classifiers.trees.RandomForest',
+ '_1_04_wctrf_1__20__01_INT_I':hp.qloguniform('_1_04_wctrf_1__20__01_INT_I', math.log(2.0), math.log(256.0), 1.0),
+ '_1_04_wctrf_1__20__02_auto_param_409':hp.choice('_1_04_wctrf_1__20__02_auto_param_409', [
+ {
+ '_1_04_wctrf_1__20__02__00__00_features_HIDDEN':'0',
+ '_1_04_wctrf_1__20__02__00__01_1_INT_K':hp.choice('_1_04_wctrf_1__20__02__00__01_1_INT_K', ['1', ]),
+ },
+ {
+ '_1_04_wctrf_1__20__02__01__00_features_HIDDEN':'1',
+ '_1_04_wctrf_1__20__02__01__01_2_INT_K':hp.qloguniform('_1_04_wctrf_1__20__02__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0),
+ },
+ ]),
+ '_1_04_wctrf_1__20__03_auto_param_412':hp.choice('_1_04_wctrf_1__20__03_auto_param_412', [
+ {
+ '_1_04_wctrf_1__20__03__00__00_depth_HIDDEN':'0',
+ '_1_04_wctrf_1__20__03__00__01_1_INT_depth':hp.choice('_1_04_wctrf_1__20__03__00__01_1_INT_depth', ['1', ]),
+ },
+ {
+ '_1_04_wctrf_1__20__03__01__00_depth_HIDDEN':'1',
+ '_1_04_wctrf_1__20__03__01__01_2_INT_depth':hp.quniform('_1_04_wctrf_1__20__03__01__01_2_INT_depth', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_04_wctrf_2_QUOTE_END':'REMOVED',
+ },
+ {
+ '_1_04_wctrt_0_QUOTE_START_B':'weka.classifiers.trees.RandomTree',
+ '_1_04_wctrt_1__21__01_INT_M':hp.qloguniform('_1_04_wctrt_1__21__01_INT_M', math.log(1.0), math.log(64.0), 1.0),
+ '_1_04_wctrt_1__21__02_U':hp.choice('_1_04_wctrt_1__21__02_U', ['REMOVE_PREV', 'REMOVED', ]),
+ '_1_04_wctrt_1__21__03_auto_param_416':hp.choice('_1_04_wctrt_1__21__03_auto_param_416', [
+ {
+ '_1_04_wctrt_1__21__03__00__00_features_HIDDEN':'0',
+ '_1_04_wctrt_1__21__03__00__01_1_INT_K':hp.choice('_1_04_wctrt_1__21__03__00__01_1_INT_K', ['0', ]),
+ },
+ {
+ '_1_04_wctrt_1__21__03__01__00_features_HIDDEN':'1',
+ '_1_04_wctrt_1__21__03__01__01_2_INT_K':hp.qloguniform('_1_04_wctrt_1__21__03__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0),
+ },
+ ]),
+ '_1_04_wctrt_1__21__04_auto_param_419':hp.choice('_1_04_wctrt_1__21__04_auto_param_419', [
+ {
+ '_1_04_wctrt_1__21__04__00__00_depth_HIDDEN':'0',
+ '_1_04_wctrt_1__21__04__00__01_1_INT_depth':hp.choice('_1_04_wctrt_1__21__04__00__01_1_INT_depth', ['0', ]),
+ },
+ {
+ '_1_04_wctrt_1__21__04__01__00_depth_HIDDEN':'1',
+ '_1_04_wctrt_1__21__04__01__01_2_INT_depth':hp.quniform('_1_04_wctrt_1__21__04__01__01_2_INT_depth', 2.0, 20.0,1.0),
+ },
+ ]),
+ '_1_04_wctrt_1__21__05_auto_param_422':hp.choice('_1_04_wctrt_1__21__05_auto_param_422', [
+ {
+ '_1_04_wctrt_1__21__05__00__00_back_HIDDEN':'0',
+ '_1_04_wctrt_1__21__05__00__01_1_INT_N':hp.choice('_1_04_wctrt_1__21__05__00__01_1_INT_N', ['0', ]),
+ },
+ {
+ '_1_04_wctrt_1__21__05__01__00_back_HIDDEN':'1',
+ '_1_04_wctrt_1__21__05__01__01_2_INT_N':hp.quniform('_1_04_wctrt_1__21__05__01__01_2_INT_N', 2.0, 5.0,1.0),
+ },
+ ]),
+ '_1_04_wctrt_2_QUOTE_END':'REMOVED',
+ },
+ ]),
+ ),
+ ]),
+ ),
+ ]),
+ ),
+ ]),
+ ),
+ ]),
+ ),
+ ),
+ ]),
+ ]),
+)
diff --git a/tests/unittests/search_spaces/nips2011.py b/tests/unittests/search_spaces/nips2011.py
new file mode 100644
index 00000000..80499bcb
--- /dev/null
+++ b/tests/unittests/search_spaces/nips2011.py
@@ -0,0 +1,56 @@
+"""
+Neural Network (NNet) and Deep Belief Network (DBN) search spaces used in [1]
+and [2].
+
+The functions in this file return pyll graphs that can be used as the `space`
+argument to e.g. `hyperopt.fmin`. The pyll graphs include hyperparameter
+constructs (e.g. `hyperopt.hp.uniform`) so `hyperopt.fmin` can perform
+hyperparameter optimization.
+
+See ./skdata_learning_algo.py for example usage of these functions.
+
+
+[1] Bergstra, J., Bardenet, R., Bengio, Y., Kegl, B. (2011). Algorithms
+for Hyper-parameter optimization, NIPS 2011.
+
+[2] Bergstra, J., Bengio, Y. (2012). Random Search for Hyper-Parameter
+Optimization, JMLR 13:281--305.
+
+"""
+
+"""
+CHANGED TO WORK AS SEARCHSPACE IN THE BBoM Framework
+"""
+
+__author__ = "James Bergstra"
+__license__ = "BSD-3"
+
+import numpy as np
+
+from hyperopt import hp
+
+space = {'preproc': hp.choice('preproc', [{
+ 'preproc' : 0}, {
+ 'preproc' : 1, # Column Normalization
+ 'colnorm_thresh' : hp.loguniform('colnorm_thresh', np.log(1e-9), np.log(1e-3)),
+ }, {
+ 'preproc' : 2, # PCA
+ 'pca_energy' : hp.uniform('pca_energy', .5, 1),
+ }]),
+ 'nhid1': hp.qloguniform('nhid1', np.log(16), np.log(1024), q=16),
+ 'dist1': hp.choice('dist1', [0, 1]), # 0 = Uniform, 1 = Normal
+ 'scale_heur1': hp.choice('scale_heur1', [{
+ 'scale_heur1' : 0, # Old
+ 'scale_mult1': hp.uniform('scale_mult1', .2, 2)}, {
+ 'scale_heur1': 1}]), # Glorot
+ 'squash' : hp.choice('squash', [0, 1]), # ['tanh', 'logistic']
+ 'iseed': hp.choice('iseed', [0, 1, 2, 3]), # Are 5, 6, 7, 8
+ 'batch_size': hp.choice('batch_size', [0, 1]), # 20 or 100
+ 'lr': hp.lognormal('lr', np.log(.01), 3.),
+ 'lr_anneal_start': hp.qloguniform('lr_anneal_start', np.log(100), np.log(10000), q=1),
+ 'l2_penalty': hp.choice('l2_penalty', [{
+ 'l2_penalty' : 0}, { # Zero
+ 'l2_penalty' : 1, # notzero
+ 'l2_penalty_nz': hp.lognormal('l2_penalty_nz', np.log(1.0e-6), 2.)}])
+ }
+
diff --git a/tests/unittests/test_benchmark_util.py b/tests/unittests/test_benchmark_util.py
new file mode 100644
index 00000000..ca1d310a
--- /dev/null
+++ b/tests/unittests/test_benchmark_util.py
@@ -0,0 +1,68 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import os
+import unittest
+import sys
+
+import HPOlib.benchmark_util as benchmark_util
+
+
+class BenchmarkUtilTest(unittest.TestCase):
+ def setUp(self):
+ # Change into the parent of the test directory
+ os.chdir(os.path.join("..", os.path.dirname(os.path.realpath(__file__))))
+
+ # Make sure there is no config file
+ try:
+ os.remove("./config.cfg")
+ except:
+ pass
+
+ def test_read_parameters_from_command_line(self):
+ # Legal call
+ sys.argv = ["test.py", "--folds", "10", "--fold", "0", "--params", "-x",
+ "3"]
+ args, params = benchmark_util.parse_cli()
+ self.assertEqual(params, {'x': '3'})
+ self.assertEqual(args, {'folds': '10', 'fold': '0'})
+
+ # illegal call, arguments with one minus before --params
+ sys.argv = ["test.py", "-folds", "10", "--fold", "0", "--params", "-x",
+ "3"]
+ with self.assertRaises(ValueError) as cm1:
+ benchmark_util.parse_cli()
+ self.assertEqual(cm1.exception.message, "You either try to use arguments"
+ " with only one leading minus or try to specify a "
+ "hyperparameter before the --params argument. test.py"
+ " -folds 10 --fold 0 --params -x 3")
+
+ # illegal call, trying to specify an arguments after --params
+ sys.argv = ["test.py", "--folds", "10", "--params", "-x",
+ "'3'", "--fold", "0"]
+ with self.assertRaises(ValueError) as cm5:
+ benchmark_util.parse_cli()
+ self.assertEqual(cm5.exception.message, "You are trying to specify an argument after the "
+ "--params argument. Please change the order.")
+
+ # illegal call, no - in front of parameter name
+ sys.argv = ["test_cv.py", "--params", "x", "'5'"]
+ with self.assertRaises(ValueError) as cm2:
+ benchmark_util.parse_cli()
+ self.assertEqual(cm2.exception.message, "Illegal command line string, expected a hyperpara"
+ "meter starting with - but found x")
\ No newline at end of file
diff --git a/tests/unittests/test_cv.py b/tests/unittests/test_cv.py
new file mode 100644
index 00000000..a05db27f
--- /dev/null
+++ b/tests/unittests/test_cv.py
@@ -0,0 +1,125 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import os
+import unittest
+import numpy as np
+import sys
+
+import HPOlib.cv as cv
+try:
+ import hyperopt
+except:
+ # TODO: Remove this Hackiness when installation fully works!
+ import HPOlib
+ hyperopt_path = os.path.join(os.path.dirname(os.path.abspath(
+ HPOlib.__file__)), "optimizers/hyperopt_august2013_mod")
+ print hyperopt_path
+ sys.path.append(hyperopt_path)
+ import hyperopt
+
+
+class CVTest(unittest.TestCase):
+ def setUp(self):
+ # Change into the parent of the test directory
+ os.chdir(os.path.join("..", os.path.dirname(os.path.realpath(__file__))))
+
+ # Make sure there is no config file
+ try:
+ os.remove("./config.cfg")
+ except:
+ pass
+
+ def test_read_parameters_from_command_line(self):
+ # Legal call
+ sys.argv = ["test_cv.py", "-x", "'5'", "-name", "'Koenigsberghausen'",
+ "-y", "'5.0'", "-z", "'-3.0'"]
+ params = cv.read_params_from_command_line()
+ self.assertEqual(params, {'x': '5', 'name': 'Koenigsberghausen',
+ 'y': '5.0', 'z': '-3.0'})
+
+ # illegal call, no - in front of parameter name
+ sys.argv = ["test_cv.py", "x", "'5'"]
+ self.assertRaises(ValueError, cv.read_params_from_command_line)
+
+ # illegal call, no single quotation mark around second parameter
+ sys.argv = ["test_cv.py", "-x", "5"]
+ self.assertRaises(ValueError, cv.read_params_from_command_line)
+
+ # illegal call, no parameter value
+ sys.argv = ["test_cv.py", "-x", "-y"]
+ self.assertRaises(ValueError, cv.read_params_from_command_line)
+
+ def test_parameter_flattening(self):
+ def naive_old_implementation(params):
+ _params_to_check = list(params.keys())
+ _new_dict = dict()
+ while len(_params_to_check) != 0:
+ p = _params_to_check.pop()
+ if isinstance(params[p], dict):
+ _params_to_check.extend(params[p].keys())
+ params.update(params[p])
+ elif isinstance(params[p], np.ndarray) or \
+ isinstance(params[p], list):
+ _new_dict[p] = params[p][0]
+ else:
+ _new_dict[p] = params[p]
+ return _new_dict
+
+ # Branin
+ import HPOlib.benchmarks.branin.tpe.space
+ space = HPOlib.benchmarks.branin.tpe.space.space
+
+ for i in range(100):
+ sample = hyperopt.pyll.stochastic.sample(space)
+ flatten = cv.flatten_parameter_dict(sample)
+ flatten_old = naive_old_implementation(sample)
+ self.assertEqual(len(flatten), 2)
+ self.assertEqual(type(flatten), dict)
+ self.assertEqual(type(flatten["x"]), float)
+ self.assertEqual(type(flatten["y"]), float)
+ self.assertEqual(flatten, flatten_old)
+
+ # HPNnet
+ import tests.search_spaces.nips2011
+ space = tests.search_spaces.nips2011.space
+
+ for i in range(100):
+ sample = hyperopt.pyll.stochastic.sample(space)
+ flatten = cv.flatten_parameter_dict(sample)
+ flatten_old = naive_old_implementation(sample)
+ # print flatten
+ # print flatten_old
+ self.assertLessEqual(len(flatten), 13)
+ self.assertGreaterEqual(len(flatten), 10)
+ self.assertEqual(type(flatten), dict)
+ self.assertEqual(flatten, flatten_old)
+
+ # AutoWEKA
+ import tests.search_spaces.autoweka
+ space = tests.search_spaces.autoweka.space
+ for i in range(100):
+ sample = hyperopt.pyll.stochastic.sample(space)
+ flatten = cv.flatten_parameter_dict(sample)
+ self.assertIn("attributesearch", flatten.keys())
+ self.assertIn("targetclass", flatten.keys())
+ # print flatten
+
+
+if __name__ == "__main__":
+ unittest.main()
\ No newline at end of file
diff --git a/tests/unittests/test_data_utils.py b/tests/unittests/test_data_utils.py
new file mode 100644
index 00000000..61831676
--- /dev/null
+++ b/tests/unittests/test_data_utils.py
@@ -0,0 +1,105 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import cPickle
+import ConfigParser
+from contextlib import contextmanager
+import numpy as np
+import os
+import sys
+import unittest
+
+import HPOlib.data_util as data_util
+
+@contextmanager
+def redirected(out=sys.stdout, err=sys.stderr):
+ saved = sys.stdout, sys.stderr
+ sys.stdout, sys.stderr = out, err
+ try:
+ yield
+ finally:
+ sys.stdout, sys.stderr = saved
+
+
+class DataUtilTest(unittest.TestCase):
+ def setUp(self):
+ # Change into the test directory
+ os.chdir(os.path.dirname(os.path.realpath(__file__)))
+
+ # Make sure there is no config file
+ try:
+ os.remove("./config.cfg")
+ except os.error:
+ pass
+
+ def tearDown(self):
+ try:
+ os.remove("./test_get_trial_index.pkl")
+ except OSError:
+ pass
+
+ def test_load_file(self):
+ # Test numpy arrays
+ train_data = np.zeros((100, 100))
+ np.save("train_data.npy", train_data)
+ data = data_util.load_file("train_data.npy", "numpy", 100)
+ self.assertTrue((train_data == data).all())
+ data = data_util.load_file("train_data.npy", "numpy", 10)
+ self.assertTrue((train_data[:10] == data).all())
+ os.remove("train_data.npy")
+
+ # Test pickle files
+ train_data = np.zeros((100, 100))
+ fh = open("train_data.pkl", "w")
+ cPickle.dump(train_data, fh)
+ fh.close()
+ data = data_util.load_file("train_data.pkl", "pickle", 100)
+ self.assertTrue((train_data == data).all())
+ data = data_util.load_file("train_data.pkl", "pickle", 10)
+ self.assertTrue((train_data[:10] == data).all())
+ os.remove("train_data.pkl")
+
+ # Test zipped pickle files
+ train_data = np.zeros((100, 100))
+ from gzip import GzipFile as gfile
+ fh = gfile("train_data.pkl.gz", "w")
+ cPickle.dump(train_data, fh)
+ fh.close()
+ data = data_util.load_file("train_data.pkl.gz", "gfile", 100)
+ self.assertTrue((train_data == data).all())
+ data = data_util.load_file("train_data.pkl.gz", "gfile", 10)
+ self.assertTrue((train_data[:10] == data).all())
+ os.remove("train_data.pkl.gz")
+
+ # Test wrong data file type
+ self.assertRaises(IOError, data_util.load_file,
+ "test.sh", "uditare", 1)
+ self.assertRaises(IOError, data_util.load_file,
+ "", "uditare", 1)
+
+ @unittest.skip("Not implemented yet")
+ def test_custom_split(self):
+ self.fail()
+
+ @unittest.skip("Not implemented yet")
+ def test_prepare_cv_for_fold(self):
+ self.fail()
+
+ @unittest.skip("Not implemented yet")
+ def test_remove_param_metadata(self):
+ self.fail()
\ No newline at end of file
diff --git a/tests/unittests/test_experiment.py b/tests/unittests/test_experiment.py
new file mode 100644
index 00000000..dfc99c3d
--- /dev/null
+++ b/tests/unittests/test_experiment.py
@@ -0,0 +1,237 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import os
+import numpy as np
+import unittest
+
+import HPOlib.Experiment as Experiment
+
+
+def _sanity_check(experiment):
+ experiment._sanity_check()
+ pass
+
+
+class ExperimentTest(unittest.TestCase):
+ # TODO: Test timing mechanism!!!
+ def setUp(self):
+ # Change into the test directory
+ os.chdir(os.path.dirname(os.path.realpath(__file__)))
+
+ try:
+ os.remove("test_exp.pkl")
+ except OSError:
+ pass
+
+ def tearDown(self):
+ # Delete the test experiment
+ try:
+ os.remove("test_exp.pkl")
+ except OSError:
+ pass
+ try:
+ os.remove("test_exp.pkl.lock")
+ except OSError:
+ pass
+
+ def test_init(self):
+ # TODO: Somehow test in which folder the experiment is created
+ # TODO: Remove the case that it is always saved automatically
+ exp = Experiment.Experiment(".", "test_exp")
+ _sanity_check(exp)
+ exp.title = "test"
+ del exp
+
+ # Make sure reloading works
+ exp = Experiment.Experiment(".", "test_exp")
+ self.assertEqual(exp.title, "test")
+ del exp
+ #self.fail()
+
+ def test_status_getters(self):
+ experiment = Experiment.Experiment(".", "test_exp", folds=2)
+ # Candidate jobs
+ experiment.add_job({"x": "0"})
+ experiment.add_job({"x": "1"})
+ # Complete jobs
+ experiment.add_job({"x": "2"})
+ experiment.set_one_fold_running(2, 0)
+ experiment.set_one_fold_complete(2, 0, 1, 1)
+ experiment.set_one_fold_running(2, 1)
+ experiment.set_one_fold_complete(2, 1, 1, 1)
+ experiment.add_job({"x": "3"})
+ experiment.set_one_fold_running(3, 0)
+ experiment.set_one_fold_complete(3, 0, 1, 1)
+ experiment.set_one_fold_running(3, 1)
+ experiment.set_one_fold_complete(3, 1, 1, 1)
+ # Incomplete jobs
+ experiment.add_job({"x": "4"})
+ experiment.set_one_fold_running(4, 0)
+ experiment.set_one_fold_complete(4, 0, 1, 1)
+ experiment.add_job({"x": "5"})
+ experiment.set_one_fold_running(5, 0)
+ experiment.set_one_fold_complete(5, 0, 1, 1)
+ # Running Jobs
+ experiment.add_job({"x": "6"})
+ experiment.set_one_fold_running(6, 0)
+ experiment.add_job({"x": "7"})
+ experiment.set_one_fold_running(7, 0)
+ # Broken Jobs
+ experiment.add_job({"x": "8"})
+ experiment.set_one_fold_running(8, 0)
+ experiment.set_one_fold_crashed(8, 0, 1000, 1)
+ experiment.add_job({"x": "9"})
+ experiment.set_one_fold_running(9, 0)
+ experiment.set_one_fold_crashed(9, 0, 1000, 1)
+ experiment.set_one_fold_running(9, 1)
+ experiment.set_one_fold_crashed(9, 1, 1000, 1)
+ self.assertEqual(len(experiment.get_candidate_jobs()), 2)
+ self.assertEqual(len(experiment.get_complete_jobs()), 2)
+ self.assertEqual(len(experiment.get_incomplete_jobs()), 3)
+ self.assertEqual(len(experiment.get_running_jobs()), 2)
+ self.assertEqual(len(experiment.get_broken_jobs()), 1)
+ self.assertEqual(experiment.trials[9]['result'], 1000)
+ self.assertNotEqual(experiment.trials[8]['result'], np.NaN)
+
+ def test_get_arg_best(self):
+ experiment = Experiment.Experiment(".", "test_exp", folds=2)
+ [experiment.add_job({"x": i}) for i in range(10)]
+ [experiment.set_one_fold_running(i, 0) for i in range(10)]
+ [experiment.set_one_fold_complete(i, 0, 10 - i, 1) for i in range(10)]
+ [experiment.set_one_fold_running(i, 1) for i in range(10)]
+ [experiment.set_one_fold_complete(i, 1, i, 1) for i in range(10)]
+ self.assertEqual(experiment.get_arg_best(), 0)
+
+ def test_get_arg_best_NaNs(self):
+ experiment = Experiment.Experiment(".", "test_exp", folds=2)
+ [experiment.add_job({"x": i}) for i in range(10)]
+ [experiment.set_one_fold_running(i, 0) for i in range(10)]
+ [experiment.set_one_fold_complete(i, 0, 10 - i, 1) for i in range(10)]
+ [experiment.set_one_fold_running(i, 1) for i in range(10)]
+ [experiment.set_one_fold_complete(i, 1, i, 1) for i in range(1, 10)]
+ self.assertEqual(experiment.get_arg_best(), 1)
+
+ def test_add_job(self):
+ exp = Experiment.Experiment(".", "test_exp", folds=5)
+ self.assertEqual(len(exp.trials), 0)
+ self.assertEqual(len(exp.instance_order), 0)
+
+ _id = exp.add_job({"x": 1, "y": 2})
+ self.assertEqual(len(exp.trials), 1)
+ self.assertEqual(len(exp.instance_order), 0)
+ self.assertEqual(exp.get_trial_from_id(_id)['instance_results'].shape,
+ (5,))
+ self.assertEqual(exp.get_trial_from_id(_id)['instance_durations'].shape,
+ (5,))
+ self.assertEqual(exp.get_trial_from_id(_id)['instance_status'].shape,
+ (5,))
+ self.assertDictEqual(exp.get_trial_from_id(_id)['params'], {"x": 1, "y": 2})
+ _sanity_check(exp)
+
+ def test_set_one_fold_crashed(self):
+ experiment = Experiment.Experiment(".", "test_exp", folds=1)
+ experiment.add_job({"x": 0})
+ experiment.set_one_fold_running(0, 0)
+ experiment.set_one_fold_crashed(0, 0, 1000, 0)
+
+ def test_one_fold_workflow(self):
+ experiment = Experiment.Experiment(".", "test_exp", folds=5)
+ trial_index = experiment.add_job({"x": 5})
+ experiment.set_one_fold_running(trial_index, 0)
+ self.assertEqual(len(experiment.get_broken_jobs()), 0)
+ self.assertEqual(len(experiment.get_complete_jobs()), 0)
+ self.assertEqual(len(experiment.get_running_jobs()), 1)
+ self.assertEqual(experiment.get_trial_from_id(trial_index)
+ ['instance_status'][0], Experiment.RUNNING_STATE)
+
+ experiment.set_one_fold_complete(trial_index, 0, 1, 1)
+ self.assertEqual(len(experiment.get_complete_jobs()), 0)
+ self.assertEqual(len(experiment.get_incomplete_jobs()), 1)
+ self.assertEqual(experiment.get_trial_from_id(trial_index)
+ ['instance_status'][0], Experiment.COMPLETE_STATE)
+
+ experiment.set_one_fold_running(trial_index, 1)
+ experiment.set_one_fold_complete(trial_index, 1, 2, 1)
+ self.assertEqual(len(experiment.get_incomplete_jobs()), 1)
+ experiment.set_one_fold_running(trial_index, 2)
+ experiment.set_one_fold_complete(trial_index, 2, 3, 1)
+ self.assertEqual(len(experiment.get_incomplete_jobs()), 1)
+ experiment.set_one_fold_running(trial_index, 3)
+ experiment.set_one_fold_complete(trial_index, 3, 4, 1)
+ self.assertEqual(len(experiment.get_incomplete_jobs()), 1)
+ experiment.set_one_fold_running(trial_index, 4)
+ experiment.set_one_fold_complete(trial_index, 4, 5, 1)
+ self.assertEqual(len(experiment.trials), 1)
+ self.assertTrue((experiment.get_trial_from_id(trial_index)["instance_results"] == [1, 2, 3, 4, 5]).all())
+ self.assertEqual(len(experiment.get_complete_jobs()), 1)
+ self.assertEqual(experiment.get_trial_from_id(trial_index)['status'],
+ Experiment.COMPLETE_STATE)
+
+ trial_index1 = experiment.add_job({"x": 6})
+ self.assertEqual(len(experiment.get_complete_jobs()), 1)
+ self.assertEqual(len(experiment.get_candidate_jobs()), 1)
+ experiment.set_one_fold_running(trial_index1, 3)
+ self.assertTrue((experiment.get_trial_from_id(trial_index1)
+ ['instance_status'] == [0, 0, 0, 2, 0]).all())
+ experiment.set_one_fold_complete(trial_index1, 3, 1, 1)
+ self.assertTrue((experiment.get_trial_from_id(trial_index1)
+ ['instance_status'] == [0, 0, 0, 3, 0]).all())
+
+ self.assertEqual(experiment.instance_order, [(0, 0), (0, 1), (0, 2),
+ (0, 3), (0, 4), (1, 3)])
+ self.assertEqual(experiment.total_wallclock_time, 6)
+ self.assertTrue((experiment.get_trial_from_id(trial_index)
+ ["instance_durations"] == [1, 1, 1, 1, 1]).all())
+
+ # Check that check_cv_finished kicked in
+ self.assertEqual(experiment.get_trial_from_id(trial_index)
+ ["result"], 3.0)
+ self.assertAlmostEqual(experiment.get_trial_from_id(trial_index)['std'], 1.4142135623730951)
+ # Check that check_cv_finished did not kick in
+ self.assertNotEqual(experiment.get_trial_from_id(trial_index1)
+ ["result"], experiment.get_trial_from_id(trial_index1)
+ ["result"])
+ self.assertNotEqual(experiment.get_trial_from_id(trial_index1)['std'],
+ experiment.get_trial_from_id(trial_index1)['std'])
+ self.assertEqual(len(experiment.trials), 2)
+ _sanity_check(experiment)
+
+ def test_remove_all_but_first_runs(self):
+ experiment = Experiment.Experiment(".", "test_exp", folds=5)
+ for i in range(5):
+ experiment.add_job({"x": i})
+ experiment.set_one_fold_running(i, i)
+ experiment.set_one_fold_complete(i, i, 1, 1)
+ experiment.set_one_fold_running(2, 3)
+ experiment.set_one_fold_complete(2, 3, 1, 1)
+
+ self.assertEqual(len(experiment.get_incomplete_jobs()), 5)
+ self.assertEqual(len(experiment.instance_order), 6)
+
+ experiment.remove_all_but_first_runs(3)
+ self.assertEqual(len(experiment.get_incomplete_jobs()), 3)
+ self.assertEqual(len(experiment.instance_order), 3)
+ self.assertTrue((experiment.get_trial_from_id(2)["instance_status"] ==
+ [0, 0, 3, 0, 0]).all())
+
+ _sanity_check(experiment)
+
+
+if __name__ == "__main__":
+ unittest.main()
\ No newline at end of file
diff --git a/tests/unittests/test_runsolver_wrapper.py b/tests/unittests/test_runsolver_wrapper.py
new file mode 100644
index 00000000..198f0fe9
--- /dev/null
+++ b/tests/unittests/test_runsolver_wrapper.py
@@ -0,0 +1,233 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import ConfigParser
+import os
+import sys
+import unittest
+
+import HPOlib.Experiment as Experiment
+import HPOlib.runsolver_wrapper as runsolver_wrapper
+
+
+class RunsolverWrapperTest(unittest.TestCase):
+ def setUp(self):
+ # Change into the test directory
+ os.chdir(os.path.dirname(os.path.realpath(__file__)))
+
+ # Make sure there is no config file
+ try:
+ os.remove("config.cfg")
+ except OSError:
+ pass
+ try:
+ os.remove("./tests.pkl")
+ except OSError:
+ pass
+ try:
+ os.remove("./tests.pkl.lock")
+ except OSError:
+ pass
+
+ def tearDown(self):
+ try:
+ os.remove("config.cfg")
+ except OSError:
+ pass
+ try:
+ os.remove("./tests.pkl")
+ except OSError:
+ pass
+ try:
+ os.remove("./tests.pkl")
+ except OSError:
+ pass
+
+ def test_read_runsolver_output(self):
+ cpu_time, wallclock_time, error = runsolver_wrapper\
+ .read_runsolver_output("runsolver_positive.txt")
+ self.assertAlmostEqual(cpu_time, 0.188011)
+ self.assertAlmostEqual(wallclock_time, 0.259524)
+ self.assertTrue(error is None)
+
+ def test_read_runsolver_output_wallclock(self):
+ cpu_time, wallclock_time, error = runsolver_wrapper\
+ .read_runsolver_output("runsolver_wallclock_time_limit.txt")
+ self.assertAlmostEqual(cpu_time, 0.044002)
+ self.assertAlmostEqual(wallclock_time, 0.066825)
+ self.assertEqual(error, "Wall clock time exceeded")
+
+ def test_read_runsolver_output_vsize(self):
+ cpu_time, wallclock_time, error = runsolver_wrapper\
+ .read_runsolver_output("runsolver_vsize_exceeded.txt")
+ self.assertAlmostEqual(cpu_time, 0.016)
+ self.assertAlmostEqual(wallclock_time,0.039276)
+ self.assertEqual(error, "VSize exceeded")
+
+ def test_read_runsolver_output_warning(self):
+ cpu_time, wallclock_time, error = runsolver_wrapper\
+ .read_runsolver_output("runsolver_positive_with_warning.txt")
+ self.assertAlmostEqual(cpu_time, 0.820027)
+ self.assertAlmostEqual(wallclock_time, 1.00203)
+ self.assertTrue(error is None)
+
+ def test_read_run_instance_output_no_result(self):
+ result_array, result_string = runsolver_wrapper.\
+ read_run_instance_output("run_instance_no_result.txt")
+ # We expect some useful output for the user
+ self.assertFalse(result_string is None)
+ self.assertTrue(result_array is None)
+
+ def test_read_run_instance_output_result(self):
+ result_array, result_string = runsolver_wrapper.\
+ read_run_instance_output("run_instance_result.txt")
+ self.assertEqual(result_string, "Result for ParamILS: SAT, 0.35, 1, "
+ "0.5, -1, Random file")
+ self.assertListEqual(result_array, ["Result", "for", "ParamILS:",
+ "SAT", "0.35", "1", "0.5",
+ "-1", "Random", "file"])
+
+ def test_get_trial_index_cv(self):
+ try:
+ os.remove("test_get_trial_index.pkl")
+ except OSError:
+ pass
+
+ try:
+ os.remove("test_get_trial_index.pkl.lock")
+ except OSError:
+ pass
+
+ experiment = Experiment.Experiment(".", "test_get_trial_index", folds=5)
+ params0 = {"x": "1"}
+ params1 = {"x": "2"}
+ params2 = {"x": "3"}
+ params3 = {"x": "4"}
+ params4 = {"x": "5"}
+
+ trial_index0 = runsolver_wrapper.get_trial_index(experiment, 0, params0)
+ self.assertEqual(trial_index0, 0)
+ experiment.set_one_fold_running(trial_index0, 0)
+ experiment.set_one_fold_complete(trial_index0, 0, 1, 1)
+ self.assertEqual(trial_index0,
+ runsolver_wrapper.get_trial_index(experiment, 1, params0))
+ experiment.set_one_fold_running(trial_index0, 1)
+ experiment.set_one_fold_complete(trial_index0, 1, 1, 1)
+ self.assertEqual(trial_index0,
+ runsolver_wrapper.get_trial_index(experiment, 2, params0))
+ experiment.set_one_fold_running(trial_index0, 2)
+ experiment.set_one_fold_complete(trial_index0, 2, 1, 1)
+ self.assertEqual(trial_index0,
+ runsolver_wrapper.get_trial_index(experiment, 3, params0))
+ experiment.set_one_fold_running(trial_index0, 3)
+ experiment.set_one_fold_complete(trial_index0, 3, 1, 1)
+ self.assertEqual(trial_index0,
+ runsolver_wrapper.get_trial_index(experiment, 4, params0))
+ experiment.set_one_fold_running(trial_index0, 4)
+ experiment.set_one_fold_complete(trial_index0, 4, 1, 1)
+
+ trial_index1 = runsolver_wrapper.get_trial_index(experiment, 0, params1)
+ self.assertEqual(trial_index1, 1)
+ experiment.set_one_fold_running(trial_index1, 0)
+ experiment.set_one_fold_complete(trial_index1, 0, 1, 1)
+ self.assertEqual(trial_index1,
+ runsolver_wrapper.get_trial_index(experiment, 1, params1))
+ experiment.set_one_fold_running(trial_index1, 1)
+ experiment.set_one_fold_complete(trial_index1, 1, 1, 1)
+ self.assertEqual(trial_index1,
+ runsolver_wrapper.get_trial_index(experiment, 2, params1))
+ experiment.set_one_fold_running(trial_index1, 2)
+ experiment.set_one_fold_complete(trial_index1, 2, 1, 1)
+ self.assertEqual(trial_index1,
+ runsolver_wrapper.get_trial_index(experiment, 3, params1))
+ experiment.set_one_fold_running(trial_index1, 3)
+ experiment.set_one_fold_complete(trial_index1, 3, 1, 1)
+ self.assertEqual(trial_index1,
+ runsolver_wrapper.get_trial_index(experiment, 4, params1))
+ experiment.set_one_fold_running(trial_index1, 4)
+ experiment.set_one_fold_complete(trial_index1, 4, 1, 1)
+
+ trial_index2 = runsolver_wrapper.get_trial_index(experiment, 0, params2)
+ self.assertEqual(trial_index2, 2)
+ experiment.set_one_fold_running(trial_index2, 0)
+ experiment.set_one_fold_complete(trial_index2, 0, 1, 1)
+
+ trial_index3 = runsolver_wrapper.get_trial_index(experiment, 0, params3)
+ self.assertEqual(trial_index3, 3)
+ experiment.set_one_fold_running(trial_index3, 0)
+ experiment.set_one_fold_complete(trial_index3, 0, 1, 1)
+
+ trial_index4 = runsolver_wrapper.get_trial_index(experiment, 0, params4)
+ self.assertEqual(trial_index4, 4)
+ experiment.set_one_fold_running(trial_index4, 0)
+ experiment.set_one_fold_complete(trial_index4, 0, 1, 1)
+
+ self.assertEqual(trial_index2,
+ runsolver_wrapper.get_trial_index(experiment, 3, params2))
+ self.assertEqual(trial_index4,
+ runsolver_wrapper.get_trial_index(experiment, 4, params4))
+
+ # Since params1 were already evaluated, this should be a new trial_index
+ trial_index_test1 = runsolver_wrapper.get_trial_index(experiment, 0, params1)
+ self.assertEqual(trial_index_test1, 5)
+
+ def test_get_trial_index_nocv(self):
+ try:
+ os.remove("test_get_trial_index.pkl")
+ except OSError:
+ pass
+
+ try:
+ os.remove("test_get_trial_index.pkl.lock")
+ except OSError:
+ pass
+
+ experiment = Experiment.Experiment(".", "test_get_trial_index", folds=1)
+ params0 = {"x": "1"}
+ params1 = {"x": "2"}
+ params2 = {"x": "3"}
+ params3 = {"x": "4"}
+ params4 = {"x": "5"}
+
+ trial_index0 = runsolver_wrapper.get_trial_index(experiment, 0, params0)
+ self.assertEqual(trial_index0, 0)
+ experiment.set_one_fold_running(trial_index0, 0)
+ experiment.set_one_fold_complete(trial_index0, 0, 1, 1)
+
+ trial_index1 = runsolver_wrapper.get_trial_index(experiment, 0, params1)
+ self.assertEqual(trial_index1, 1)
+ experiment.set_one_fold_running(trial_index1, 0)
+ experiment.set_one_fold_complete(trial_index1, 0, 1, 1)
+
+ trial_index2 = runsolver_wrapper.get_trial_index(experiment, 0, params2)
+ self.assertEqual(trial_index2, 2)
+ experiment.set_one_fold_running(trial_index2, 0)
+ experiment.set_one_fold_complete(trial_index2, 0, 1, 1)
+
+ trial_index3 = runsolver_wrapper.get_trial_index(experiment, 0, params3)
+ self.assertEqual(trial_index3, 3)
+ experiment.set_one_fold_running(trial_index3, 0)
+ experiment.set_one_fold_complete(trial_index3, 0, 1, 1)
+
+ trial_index4 = runsolver_wrapper.get_trial_index(experiment, 0, params4)
+ self.assertEqual(trial_index4, 4)
+ experiment.set_one_fold_running(trial_index4, 0)
+ experiment.set_one_fold_complete(trial_index4, 0, 1, 1)
+
+ self.assertEqual(5,
+ runsolver_wrapper.get_trial_index(experiment, 0, params2))
diff --git a/tests/unittests/test_wrapping.py b/tests/unittests/test_wrapping.py
new file mode 100644
index 00000000..1b766ab6
--- /dev/null
+++ b/tests/unittests/test_wrapping.py
@@ -0,0 +1,68 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import os
+import shutil
+import sys
+import unittest
+import tempfile
+import StringIO
+
+import HPOlib.wrapping as wrapping
+import HPOlib.config_parser.parse as parse
+
+class WrappingTest(unittest.TestCase):
+ def setUp(self):
+ # Change into the test directory
+ os.chdir(os.path.dirname(os.path.realpath(__file__)))
+
+ # Make sure there is no config file
+ try:
+ os.remove("./config.cfg")
+ except:
+ pass
+
+ @unittest.skip("Not implemented yet")
+ def test_calculate_wrapping_overhead(self):
+ self.fail()
+
+ @unittest.skip("Not implemented yet")
+ def test_calculate_optimizer_time(self):
+ self.fail()
+
+ def test_use_option_parser_no_optimizer(self):
+ # Test not specifying an optimizer but random other options
+ sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet']
+ self.assertRaises(SystemExit, wrapping.use_arg_parser)
+
+ def test_use_option_parser_the_right_way(self):
+ sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet', '-o', 'SMAC']
+ args, unknown = wrapping.use_arg_parser()
+ self.assertEqual(args.optimizer, 'SMAC')
+ self.assertEqual(args.seed, 1)
+ self.assertEqual(args.title, 'DBNet')
+ self.assertEqual(len(unknown), 0)
+
+ # General main test
+ @unittest.skip("Not implemented yet")
+ def test_main(self):
+ self.fail()
+
+
+if __name__ == "__main__":
+ unittest.main()
\ No newline at end of file
diff --git a/tests/unittests/test_wrapping_util.py b/tests/unittests/test_wrapping_util.py
new file mode 100644
index 00000000..2d927149
--- /dev/null
+++ b/tests/unittests/test_wrapping_util.py
@@ -0,0 +1,102 @@
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import ConfigParser
+import numpy as np
+import os
+import shutil
+import sys
+import unittest
+import tempfile
+import StringIO
+
+import HPOlib.wrapping as wrapping
+import HPOlib.wrapping_util as wrapping_util
+import HPOlib.config_parser.parse as parse
+
+
+class WrappingTestUtil(unittest.TestCase):
+ def setUp(self):
+ # Change into the test directory
+ os.chdir(os.path.dirname(os.path.realpath(__file__)))
+
+ # Make sure there is no config file
+ # noinspection PyBroadException
+ try:
+ os.remove("./config.cfg")
+ except:
+ pass
+
+ def test_save_config_to_file(self):
+ config = parse.parse_config("dummy_config.cfg", allow_no_value=True)
+ config.set("HPOLIB", "total_time_limit", None)
+ string_stream = StringIO.StringIO()
+ wrapping_util.save_config_to_file(string_stream, config)
+ file_content = string_stream.getvalue()
+ asserted_file_content = "[HPOLIB]\n" \
+ "number_of_jobs = 1\n" \
+ "result_on_terminate = 1\n"\
+ "function = 1\n"\
+ "total_time_limit = None\n"\
+ "[GRIDSEARCH]\n" \
+ "params = params.pcs\n"
+
+ self.assertEqual(asserted_file_content, file_content)
+ string_stream.close()
+
+ def test_save_config_to_file_ignore_none(self):
+ config = parse.parse_config("dummy_config.cfg", allow_no_value=True)
+ config.set("HPOLIB", "total_time_limit", None)
+ string_stream = StringIO.StringIO()
+ wrapping_util.save_config_to_file(string_stream, config,
+ write_nones=False)
+ file_content = string_stream.getvalue()
+ asserted_file_content = "[HPOLIB]\n" \
+ "number_of_jobs = 1\n" \
+ "result_on_terminate = 1\n"\
+ "function = 1\n"\
+ "[GRIDSEARCH]\n" \
+ "params = params.pcs\n"
+
+ self.assertEqual(asserted_file_content, file_content)
+ string_stream.close()
+
+ def test_use_option_parser_with_config(self):
+ sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet', '-o', 'SMAC',
+ '--HPOLIB:number_of_jobs', '2']
+ args, unknown = wrapping.use_arg_parser()
+ self.assertEqual(len(unknown), 2)
+ config = ConfigParser.SafeConfigParser(allow_no_value=True)
+ config.read("dummy_config.cfg")
+ config_args = wrapping_util.parse_config_values_from_unknown_arguments(
+ unknown, config)
+ self.assertEqual(vars(config_args)['HPOLIB:number_of_jobs'], '2')
+
+ def test_nan_mean(self):
+ self.assertEqual(wrapping_util.nan_mean(np.array([1, 5])), 3)
+ self.assertEqual(wrapping_util.nan_mean((1, 5)), 3)
+ # self.assertEqual(wrapping_util.nan_mean("abc"), 3)
+ self.assertRaises(TypeError, wrapping_util.nan_mean, ("abc"))
+ self.assertEqual(wrapping_util.nan_mean(np.array([1, 5, np.nan])), 3)
+ self.assertEqual(wrapping_util.nan_mean(np.array([1, 5, np.inf])), 3)
+ self.assertTrue(np.isnan(wrapping_util.nan_mean(np.array([np.inf]))))
+ self.assertEqual(wrapping_util.nan_mean(np.array([-1, 1])), 0)
+ self.assertTrue(np.isnan(wrapping_util.nan_mean(np.array([]))))
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tpe.py b/tpe.py
deleted file mode 100644
index d2274caa..00000000
--- a/tpe.py
+++ /dev/null
@@ -1,125 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-#!/usr/bin/env python
-
-import cPickle
-import os
-import sys
-
-import wrapping_util
-
-
-version_info = ("# %76s #\n" % "https://github.com/hyperopt/hyperopt/tree/486aebec8a4170e4781d99bbd6cca09123b12717")
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__contact__ = "automl.org"
-
-
-def build_tpe_call(config, options, optimizer_dir):
- #For TPE we have to cd to the exp_dir
- call = "cd " + optimizer_dir + "\n" + \
- "python " + os.path.dirname(os.path.realpath(__file__)) + \
- "/tpecall.py"
- call = ' '.join([call, '-p', config.get('TPE', 'space'),
- "-a", config.get('DEFAULT', 'algorithm'),
- "-m", config.get('TPE', 'numberEvals'),
- "-s", str(options.seed)])
- if options.restore:
- call = ' '.join([call, '-r'])
- return call
-
-
-#noinspection PyUnusedLocal
-def restore(config, optimizer_dir, **kwargs):
- """
- Returns the number of restored runs. This is the number of different configs
- tested multiplied by the number of crossvalidation folds.
- """
- restore_file = os.path.join(optimizer_dir, 'state.pkl')
- if not os.path.exists(restore_file):
- print "Oups, this should have been checked before"
- raise Exception("%s does not exist" % (restore_file,))
-
- # Special settings for restoring
- fh = open(restore_file)
- state = cPickle.load(fh)
- fh.close()
- complete_runs = 0
- #noinspection PyProtectedMember
- tpe_trials = state['trials']._trials
- for trial in tpe_trials:
- # Assumes that all not valid states states are marked crashed
- if trial['state'] == 2:
- complete_runs += 1
- restored_runs = complete_runs * config.getint('DEFAULT', 'numberCV')
- return restored_runs
-
-
-#noinspection PyUnusedLocal
-def main(config, options, experiment_dir, **kwargs):
- # config: Loaded .cfg file
- # options: Options containing seed, restore_dir,
- # experiment_dir: Experiment directory/Benchmark_directory
- # **kwargs: Nothing so far
- time_string = wrapping_util.get_time_string()
- cmd = ""
- SYSTEM_WIDE = 0
- AUGUST_2013_MOD = 1
-
- try:
- import hyperopt
- version = SYSTEM_WIDE
- except ImportError:
- cmd += "export PYTHONPATH=$PYTHONPATH:" + os.path.dirname(os.path.abspath(__file__)) + \
- "/optimizers/hyperopt_august2013_mod/hyperopt"
- import optimizers.spearmint_april2013_mod.hyperopt as hyperopt
- version = AUGUST_2013_MOD
-
- path_to_optimizer = os.path.abspath(os.path.dirname(hyperopt.__file__))
-
- # Find experiment directory
- if options.restore:
- if not os.path.exists(options.restore):
- raise Exception("The restore directory does not exist")
- optimizer_dir = options.restore
- else:
- optimizer_dir = os.path.join(experiment_dir, "tpe_" +
- str(options.seed) + "_" +
- time_string)
-
- # Build call
- cmd += build_tpe_call(config, options, optimizer_dir)
-
- # Set up experiment directory
- if not os.path.exists(optimizer_dir):
- os.mkdir(optimizer_dir)
- space = config.get('TPE', 'space')
- # Copy the hyperopt search space
- if not os.path.exists(os.path.join(optimizer_dir, space)):
- os.symlink(os.path.join(experiment_dir, "tpe", space),
- os.path.join(optimizer_dir, space))
- sys.stdout.write("### INFORMATION ################################################################\n")
- sys.stdout.write("# You're running %40s #\n" % path_to_optimizer)
- if version == SYSTEM_WIDE:
- pass
- else:
- sys.stdout.write("# To reproduce our results you need version 0.0.3.dev, which can be found here:#\n")
- sys.stdout.write("%s" % version_info)
- sys.stdout.write("# A newer version might be available, but not yet built in. #\n")
- sys.stdout.write("################################################################################\n")
- return cmd, optimizer_dir
\ No newline at end of file
diff --git a/wrapping.py b/wrapping.py
deleted file mode 100644
index 42c8c405..00000000
--- a/wrapping.py
+++ /dev/null
@@ -1,223 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-#!/usr/bin/env python
-
-import imp
-import numpy as np
-import os
-import subprocess
-import sys
-import time
-
-import check_before_start
-from config_parser.parse import parse_config
-import Experiment
-import numpy as np
-from optparse import OptionParser
-
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__contact__ = "automl.org"
-
-
-def calculate_wrapping_overhead(trials):
- wrapping_time = 0
- for times in zip(trials.cv_starttime, trials.cv_endtime):
- wrapping_time += times[1] - times[0]
-
- wrapping_time = wrapping_time - np.nansum(trials.instance_durations)
- return wrapping_time
-
-
-def calculate_optimizer_time(trials):
- optimizer_time = []
- time_idx = 0
-
- optimizer_time.append(trials.cv_starttime[0] - trials.starttime[time_idx])
-
- for i in range(len(trials.cv_starttime[1:])):
- if trials.cv_starttime[i + 1] > trials.endtime[time_idx]:
- optimizer_time.append(trials.endtime[time_idx] -
- trials.cv_endtime[i])
- time_idx += 1
- optimizer_time.append(trials.cv_starttime[i + 1] -
- trials.starttime[time_idx])
- else:
- optimizer_time.append(trials.cv_starttime[i + 1] -
- trials.cv_endtime[i])
-
- optimizer_time.append(trials.endtime[time_idx] - trials.cv_endtime[-1])
- trials.optimizer_time = optimizer_time
-
- return np.nansum(optimizer_time)
-
-
-def main():
- # Parse options and arguments
- parser = OptionParser()
- parser.set_usage("wrapping.py [-s seed] [-t title]" +
- "[--restore=/directory/]")
- parser.add_option("-p", "--print", action="store_true", dest="printcmd",
- default=False,
- help="If set print the command instead of executing it")
- parser.add_option("-s", "--seed", dest="seed", default=1, type=int,
- help="Set the seed of the optimizer")
- parser.add_option("-t", "--title", dest="title", default=None,
- help="A title for the experiment")
- restore_help_string = "Restore the state from a given directory"
- parser.add_option("--restore", default=None, dest="restore",
- help=restore_help_string)
-
- (options, args) = parser.parse_args()
-
- # find out which optimizer we are supposed to use
- if len(args) < 1:
- parser.print_help(file=sys.stderr)
- sys.exit(1)
-
- optimizer = args[0]
-
- experiment_dir = os.getcwd()
-
- # _check_runsolver, _check_modules(), _check_config(experiment_dir)
- check_before_start._check_first(experiment_dir)
-
- # build call
- config_file = os.path.join(experiment_dir, "config.cfg")
- config = parse_config(config_file, allow_no_value=True, optimizer_module=optimizer)
- wrapping_dir = os.path.dirname(os.path.realpath(__file__))
- cmd = "export PYTHONPATH=$PYTHONPATH:" + wrapping_dir + "\n"
-
- # Load optimizer
- try:
- optimizer_module = imp.load_source(optimizer, wrapping_dir + "/" +
- optimizer + ".py")
- except (ImportError, IOError):
- print "Optimizer module", optimizer, "not found"
- import traceback
- print traceback.format_exc()
- sys.exit(1)
- optimizer_call, optimizer_dir = optimizer_module.main(config=config,
- options=options,
- experiment_dir=
- experiment_dir)
- cmd += optimizer_call
-
- # _check_function
- check_before_start._check_second(experiment_dir, optimizer_dir)
-
- # initialize/reload pickle file
- if options.restore:
- try:
- os.remove(os.path.join(optimizer_dir, optimizer + ".pkl.lock"))
- except OSError:
- pass
- folds = config.getint('DEFAULT', 'numberCV')
- trials = Experiment.Experiment(optimizer_dir, optimizer, folds=folds,
- max_wallclock_time=config.get('DEFAULT',
- 'cpu_limit'),
- title=options.title)
- trials.optimizer = optimizer
-
- if options.restore:
- # Old versions did store NaNs instead of the worst possible result for
- # crashed runs in the instance_results. In order to be able to load
- # these files, these NaNs are replaced
- for i, trial in enumerate(trials.instance_order):
- _id, fold = trial
- instance_result = trials.get_trial_from_id(_id)['instance_results'][fold]
- if not np.isfinite(instance_result):
- # Make sure that we do delete the last job which was running but
- # did not finish
- if i == len(trials.instance_order) - 1 and \
- len(trials.cv_starttime) != len(trials.cv_endtime):
- # The last job obviously did not finish correctly, do not
- # replace it
- pass
- else:
- trials.get_trial_from_id(_id)['instance_results'][fold] = \
- config.getfloat('DEFAULT', 'result_on_terminate')
- # Pretty sure we need this here:
- trials.get_trial_from_id(_id)['instance_status'][fold] = \
- Experiment.BROKEN_STATE
-
- #noinspection PyBroadException
- try:
- restored_runs = optimizer_module.restore(config=config,
- optimizer_dir=optimizer_dir,
- cmd=cmd)
- except:
- print "Could not restore runs for %s" % options.restore
- import traceback
- print traceback.format_exc()
- sys.exit(1)
-
- print "Restored %d runs" % restored_runs
- trials.remove_all_but_first_runs(restored_runs)
- fh = open(os.path.join(optimizer_dir, optimizer + ".out"), "a")
- fh.write("#" * 80 + "\n" + "Restart! Restored %d runs.\n" % restored_runs)
- fh.close()
-
- if len(trials.endtime) < len(trials.starttime):
- trials.endtime.append(trials.cv_endtime[-1])
- trials.starttime.append(time.time())
- else:
- trials.starttime.append(time.time())
- #noinspection PyProtectedMember
- trials._save_jobs()
- del trials
- sys.stdout.flush()
-
- # Run call
- if options.printcmd:
- print cmd
- return 0
- else:
- print cmd
- output_file = os.path.join(optimizer_dir, optimizer + ".out")
- fh = open(output_file, "a")
- process = subprocess.Popen(cmd, stdout=fh, stderr=fh, shell=True,
- executable="/bin/bash")
- print "-----------------------RUNNING----------------------------------"
- ret = process.wait()
- trials = Experiment.Experiment(optimizer_dir, optimizer)
- trials.endtime.append(time.time())
- #noinspection PyProtectedMember
- trials._save_jobs()
- # trials.finish_experiment()
- print "Finished with return code: " + str(ret)
- total_time = 0
- print trials.get_best()
- #noinspection PyBroadException
- try:
- for starttime, endtime in zip(trials.starttime, trials.endtime):
- total_time += endtime - starttime
- print "Needed a total of %f seconds" % total_time
- print "The optimizer %s took %f seconds" % \
- (optimizer, calculate_optimizer_time(trials))
- print "The overhead of the wrapping software is %f seconds" % \
- (calculate_wrapping_overhead(trials))
- print "The algorithm itself took %f seconds" % \
- trials.total_wallclock_time
- except:
- pass
- del trials
- return ret
-
-if __name__ == "__main__":
- main()
\ No newline at end of file
diff --git a/wrapping_util.py b/wrapping_util.py
deleted file mode 100644
index 4de3f190..00000000
--- a/wrapping_util.py
+++ /dev/null
@@ -1,73 +0,0 @@
-##
-# wrapping: A program making it easy to use hyperparameter
-# optimization software.
-# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-import datetime
-import math
-import traceback
-
-__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
-__contact__ = "automl.org"
-
-
-def get_time_string():
- local_time = datetime.datetime.today()
- time_string = "%d-%d-%d--%d-%d-%d-%d" % (local_time.year, local_time.month,
- local_time.day, local_time.hour, local_time.minute,
- local_time.second, local_time.microsecond)
- return time_string
-
-
-def float_eq( a, b, eps=0.0001 ):
- return abs(math.log( a ) - math.log(b)) <= eps
-
-
-def format_traceback(exc_info):
- traceback_template = '''Traceback (most recent call last):
- File "%(filename)s", line %(lineno)s, in %(name)s
- %(type)s: %(message)s\n''' # Skipping the "actual line" item
-
- # Also note: we don't walk all the way through the frame stack in this example
- # see hg.python.org/cpython/file/8dffb76faacc/Lib/traceback.py#l280
- # (Imagine if the 1/0, below, were replaced by a call to test() which did 1/0.)
-
- exc_type, exc_value, exc_traceback = exc_info # most recent (if any) by default
-
- '''
- Reason this _can_ be bad: If an (unhandled) exception happens AFTER this,
- or if we do not delete the labels on (not much) older versions of Py, the
- reference we created can linger.
-
- traceback.format_exc/print_exc do this very thing, BUT note this creates a
- temp scope within the function.
- '''
-
- traceback_details = {
- 'filename': exc_traceback.tb_frame.f_code.co_filename,
- 'lineno' : exc_traceback.tb_lineno,
- 'name' : exc_traceback.tb_frame.f_code.co_name,
- 'type' : exc_type.__name__,
- 'message' : exc_value.message, # or see traceback._some_str()
- }
-
- del(exc_type, exc_value, exc_traceback) # So we don't leave our local labels/objects dangling
- # This still isn't "completely safe", though!
- # "Best (recommended) practice: replace all exc_type, exc_value, exc_traceback
- # with sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]
-
- return "\n" + traceback.format_exc() + "\n\n" + traceback_template % traceback_details + "\n\n"
-