From 3c0c225580f9f06fe64aa683952eb5b122886f5e Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 22 Jan 2014 11:23:58 +0100 Subject: [PATCH 001/139] modify .gitignore ignore experiment data, optimizer sources and runsolver sources --- .gitignore | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitignore b/.gitignore index ded60678..0210d597 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,16 @@ *.py[cod] +# Optimizers +optimizers/smac_2_06_01-dev/* +optimizers/spearmint_april2013_mod/* +optimizers/hyperopt_august2013_mod/* + +# Runsolver +runsolver/src/* + +# benchmark runs +benchmarks/*/*_*/* + # C extensions *.so From 5f54ac236e78a53df2fb1dc413676a61b5b3677c Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 23 Jan 2014 18:40:28 +0100 Subject: [PATCH 002/139] Refactor: changed config parser exception --- config_parser/parse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config_parser/parse.py b/config_parser/parse.py index ae82263b..2d822d67 100644 --- a/config_parser/parse.py +++ b/config_parser/parse.py @@ -29,7 +29,8 @@ def parse_config(config_fn, allow_no_value=True, optimizer_module=""): # Overwrites with default values from generalDefault.cfg # Loads optimizer specific parser, called 'optimizer_module'_parser.py, which can read its own default config if not os.path.isfile(config_fn): - raise Exception('%s is not a valid file\n' % config_fn) + raise Exception('%s is not a valid file\n' % os.path.join( + os.getcwd(), config_fn)) config = ConfigParser.SafeConfigParser(allow_no_value=allow_no_value) config.read(config_fn) From d2359ff7ed5b761d93ecb089b99ce67d0458ee5a Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 23 Jan 2014 18:45:48 +0100 Subject: [PATCH 003/139] Fix: Corrected hyperopt import statement in tpe.py --- tpe.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tpe.py b/tpe.py index d2274caa..1004754e 100644 --- a/tpe.py +++ b/tpe.py @@ -86,8 +86,8 @@ def main(config, options, experiment_dir, **kwargs): version = SYSTEM_WIDE except ImportError: cmd += "export PYTHONPATH=$PYTHONPATH:" + os.path.dirname(os.path.abspath(__file__)) + \ - "/optimizers/hyperopt_august2013_mod/hyperopt" - import optimizers.spearmint_april2013_mod.hyperopt as hyperopt + "/optimizers/hyperopt_august2013_mod\n" + import optimizers.hyperopt_august2013_mod.hyperopt as hyperopt version = AUGUST_2013_MOD path_to_optimizer = os.path.abspath(os.path.dirname(hyperopt.__file__)) @@ -122,4 +122,4 @@ def main(config, options, experiment_dir, **kwargs): sys.stdout.write("%s" % version_info) sys.stdout.write("# A newer version might be available, but not yet built in. #\n") sys.stdout.write("################################################################################\n") - return cmd, optimizer_dir \ No newline at end of file + return cmd, optimizer_dir From 7eec3bf177b58d02078f5de749475a0cf272c226 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 23 Jan 2014 18:47:43 +0100 Subject: [PATCH 004/139] Fix: removed unnecessary character --- format_converter/TpeSMAC.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/format_converter/TpeSMAC.py b/format_converter/TpeSMAC.py index 7da602c7..df3cefa7 100644 --- a/format_converter/TpeSMAC.py +++ b/format_converter/TpeSMAC.py @@ -57,7 +57,7 @@ def __repr__(self): EQ = partial(Cond, op='=') -def expr_to_config(expr, conditions, hps):z +def expr_to_config(expr, conditions, hps): if conditions is None: conditions = () assert isinstance(expr, hyperopt.pyll.base.Apply) From be178e7097dcea6ecb42bb6df18826ce6c092f4f Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 23 Jan 2014 18:52:24 +0100 Subject: [PATCH 005/139] Refactor: Better error message for config loading in run_instance.py --- run_instance.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/run_instance.py b/run_instance.py index ee577942..6238d7d2 100644 --- a/run_instance.py +++ b/run_instance.py @@ -179,11 +179,13 @@ def main(): fn_name, ext = os.path.splitext(os.path.basename(fn_path)) try: fn = imp.load_source(fn_name, fn_path) - except (ImportError, IOError): + except (ImportError, IOError) as e: + print "Raised", e, "trying to recover..." try: + print os.path.exists(fn_path_parent) fn = imp.load_source(fn_name, fn_path_parent) except (ImportError, IOError): - print os.path.join(fn_path, "..") + print os.path.join(fn_path_parent) print(("Could not find\n%s\n\tin\n%s\n\tor its parent directory " + "relative to\n%s") % (fn_name, fn_path, os.getcwd())) From c9723e176dc351f745f7f4f81e8209150f68bbfe Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 23 Jan 2014 19:29:54 +0100 Subject: [PATCH 006/139] Refactor: Issue #16: Replace optparse by argparse --- check_before_start.py | 13 ++++---- wrapping.py | 69 +++++++++++++++++++++++++------------------ wrapping_util.py | 20 +++++++++++++ 3 files changed, 68 insertions(+), 34 deletions(-) diff --git a/check_before_start.py b/check_before_start.py index d965dd9d..142957cc 100644 --- a/check_before_start.py +++ b/check_before_start.py @@ -55,7 +55,7 @@ def _check_modules(): % scipy.__version__ except: raise ImportError("Scipy cannot be imported. Are you sure that it's installed?") - import bson + #import bson import networkx import google.protobuf @@ -65,7 +65,7 @@ def _check_modules(): print "Theano not found" if 'cuda' not in os.environ['PATH']: - print "CUDA not in $PATH" + print "\tCUDA not in $PATH" # if 'cuda' not in os.environ['LD_LIBRARY_PATH']: # print "CUDA not in $LD_LIBRARY_PATH" @@ -117,14 +117,17 @@ def _check_function(experiment_dir, optimizer_dir): del fn -def _check_first(experiment_dir): - """ Do some checks before optimizer is loaded """ - main() +def _check_zeroth(experiment_dir): print "\tconfig.cfg..", _check_config(experiment_dir) print "..passed" +def _check_first(experiment_dir): + """ Do some checks before optimizer is loaded """ + main() + + def _check_second(experiment_dir, optimizer_dir): """ Do remaining tests """ print "\talgorithm..", diff --git a/wrapping.py b/wrapping.py index 42c8c405..e1c1ad69 100644 --- a/wrapping.py +++ b/wrapping.py @@ -29,7 +29,8 @@ from config_parser.parse import parse_config import Experiment import numpy as np -from optparse import OptionParser +import argparse +from argparse import ArgumentParser __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" @@ -64,42 +65,52 @@ def calculate_optimizer_time(trials): optimizer_time.append(trials.endtime[time_idx] - trials.cv_endtime[-1]) trials.optimizer_time = optimizer_time - return np.nansum(optimizer_time) - - -def main(): - # Parse options and arguments - parser = OptionParser() - parser.set_usage("wrapping.py [-s seed] [-t title]" + - "[--restore=/directory/]") - parser.add_option("-p", "--print", action="store_true", dest="printcmd", + return np.nansum(optimizer_time) + + +def use_option_parser(): + """Parse all options which can be handled by the wrapping script. + Unknown arguments are ignored and returned as a list. It is useful to + check this list in your program to handle typos etc. + + Returns: + a tuple. The first element is an argparse.Namespace object, + the second a list with all unknown arguments. + """ + parser = ArgumentParser(description="Perform an experiment with the " + "HPOlib") + parser.add_argument("-o", "--optimizer", action="store", type=str, + help="Specify the optimizer name.", required=True) + parser.add_argument("-p", "--print", action="store_true", dest="printcmd", default=False, help="If set print the command instead of executing it") - parser.add_option("-s", "--seed", dest="seed", default=1, type=int, + parser.add_argument("-s", "--seed", dest="seed", default=1, type=int, help="Set the seed of the optimizer") - parser.add_option("-t", "--title", dest="title", default=None, + parser.add_argument("-t", "--title", dest="title", default=None, help="A title for the experiment") restore_help_string = "Restore the state from a given directory" - parser.add_option("--restore", default=None, dest="restore", + parser.add_argument("--restore", default=None, dest="restore", help=restore_help_string) - (options, args) = parser.parse_args() + args = parser.parse_args() + return args - # find out which optimizer we are supposed to use - if len(args) < 1: - parser.print_help(file=sys.stderr) - sys.exit(1) - - optimizer = args[0] +def main(): + """Start an optimization of the HPOlib. For documentation see the + comments inside this function and the general HPOlib documentation.""" experiment_dir = os.getcwd() + check_before_start._check_zeroth(experiment_dir) + args = use_option_parser() + optimizer = args.optimizer - # _check_runsolver, _check_modules(), _check_config(experiment_dir) + config_file = os.path.join(experiment_dir, "config.cfg") + config = parse_config(config_file, allow_no_value=True, optimizer_module=optimizer) + + # _check_runsolver, _check_modules() check_before_start._check_first(experiment_dir) # build call - config_file = os.path.join(experiment_dir, "config.cfg") - config = parse_config(config_file, allow_no_value=True, optimizer_module=optimizer) wrapping_dir = os.path.dirname(os.path.realpath(__file__)) cmd = "export PYTHONPATH=$PYTHONPATH:" + wrapping_dir + "\n" @@ -113,7 +124,7 @@ def main(): print traceback.format_exc() sys.exit(1) optimizer_call, optimizer_dir = optimizer_module.main(config=config, - options=options, + options=args, experiment_dir= experiment_dir) cmd += optimizer_call @@ -122,7 +133,7 @@ def main(): check_before_start._check_second(experiment_dir, optimizer_dir) # initialize/reload pickle file - if options.restore: + if args.restore: try: os.remove(os.path.join(optimizer_dir, optimizer + ".pkl.lock")) except OSError: @@ -131,10 +142,10 @@ def main(): trials = Experiment.Experiment(optimizer_dir, optimizer, folds=folds, max_wallclock_time=config.get('DEFAULT', 'cpu_limit'), - title=options.title) + title=args.title) trials.optimizer = optimizer - if options.restore: + if args.restore: # Old versions did store NaNs instead of the worst possible result for # crashed runs in the instance_results. In order to be able to load # these files, these NaNs are replaced @@ -162,7 +173,7 @@ def main(): optimizer_dir=optimizer_dir, cmd=cmd) except: - print "Could not restore runs for %s" % options.restore + print "Could not restore runs for %s" % args.restore import traceback print traceback.format_exc() sys.exit(1) @@ -184,7 +195,7 @@ def main(): sys.stdout.flush() # Run call - if options.printcmd: + if args.printcmd: print cmd return 0 else: diff --git a/wrapping_util.py b/wrapping_util.py index 4de3f190..b3b78bcd 100644 --- a/wrapping_util.py +++ b/wrapping_util.py @@ -19,6 +19,10 @@ import datetime import math import traceback +import os +import sys + +from config_parser.parse import parse_config __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" @@ -71,3 +75,19 @@ def format_traceback(exc_info): return "\n" + traceback.format_exc() + "\n\n" + traceback_template % traceback_details + "\n\n" + +def load_experiment_config_file(): + # Load the config file, this holds information about data, black box fn etc. + try: + cfg_filename = "config.cfg" + cfg = parse_config(cfg_filename, allow_no_value=True) + if not cfg.has_option("DEFAULT", "is_not_original_config_file"): + print "Config file in directory %s seems to be an original config "\ + "which was not created by wrapping.py. Please contact the " \ + "HPOlib maintainer to solve this issue." + sys.exit(1) + return cfg + except IOError as e: + print "Could not open config file in directory %s" % os.getcwd() + sys.exit(1) + From 9b46421c10de08bfa07118388f3d3b7b10519b0d Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 23 Jan 2014 19:38:06 +0100 Subject: [PATCH 007/139] We can now override config.cfg Overriding config is possible for parameters from the DEFAULT section. I will open a new issue so that it will be possible to override parameters from the other sections. --- cv.py | 21 +++--------- runsolver_wrapper.py | 16 ++-------- wrapping.py | 76 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 79 insertions(+), 34 deletions(-) diff --git a/cv.py b/cv.py index fd417141..18d648c7 100644 --- a/cv.py +++ b/cv.py @@ -28,10 +28,9 @@ import numpy as np -from config_parser.parse import parse_config from Experiment import Experiment -from run_instance import run_instance from wrapping_util import format_traceback +from wrapping_util import load_experiment_config_file __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __license__ = "3-clause BSD License" @@ -48,14 +47,7 @@ def doCV(params, folds=10): print "Starting Cross validation" sys.stdout.flush() optimizer = os.getcwd().split("/")[-1].split("_")[0] - - # Load the config file, this holds information about data, black box fn etc. - try: - cfg_filename = "config.cfg" - cfg = parse_config(cfg_filename, allow_no_value=True) - except: - cfg_filename = "../config.cfg" - cfg = parse_config(cfg_filename, allow_no_value=True) + cfg = load_experiment_config_file() # Now evaluate $fold times if folds < 1: return np.NaN @@ -202,13 +194,8 @@ def main(job_id, params): experiment.start_cv(cv_starttime) del experiment - # Load the config file, this holds information about data, black box fn etc. - try: - cfg_filename = "config.cfg" - cfg = parse_config(cfg_filename, allow_no_value=True) - except: - cfg_filename = "../config.cfg" - cfg = parse_config(cfg_filename, allow_no_value=True) + cfg_filename = "config.cfg" + cfg = load_experiment_config_file() # Load number of folds folds = cfg.getint('DEFAULT', 'numberCV') diff --git a/runsolver_wrapper.py b/runsolver_wrapper.py index 1900198b..7290233a 100644 --- a/runsolver_wrapper.py +++ b/runsolver_wrapper.py @@ -25,7 +25,6 @@ import time import subprocess -from config_parser.parse import parse_config import Experiment import wrapping_util @@ -107,18 +106,6 @@ def read_run_instance_output(run_instance_output): return result_array, result_string -def get_config(): - cfg_filename = "config.cfg" - if not os.path.isfile(cfg_filename): - cfg_filename = "../config.cfg" - if not os.path.isfile(cfg_filename): - raise Exception('Could not find config.cfg in ./ or .// from %s\n' - % os.getcwd()) - - cfg = parse_config(cfg_filename, allow_no_value=True) - return cfg, cfg_filename - - def main(): # Parse options and arguments usage = "This script pickles the params and runs the runsolver with " +\ @@ -145,7 +132,8 @@ def main(): del experiment # Again we need to find the config.cfg - cfg, cfg_filename = get_config() + cfg = cfg = wrapping_util.load_experiment_config_file() + cfg_filename = "config.cfg" # Ignore smac cutofftime time_limit = cfg.getint('DEFAULT', 'runsolver_time_limit') diff --git a/wrapping.py b/wrapping.py index e1c1ad69..d52bb0a0 100644 --- a/wrapping.py +++ b/wrapping.py @@ -92,8 +92,68 @@ def use_option_parser(): parser.add_argument("--restore", default=None, dest="restore", help=restore_help_string) - args = parser.parse_args() - return args + args, unknown = parser.parse_known_args() + return args, unknown + + +def parse_config_values_from_unknown_arguments(unknown_arguments, config): + """Parse values not recognized by use_option_parser for config values. + + Args: + unknown_arguments: A list of arguments which is returned by + use_option_parser. It should only contain keys which are allowed + in config files. + config: A ConfigParser.SafeConfigParser object which contains all keys + should be parsed from the unknown_arguments list. + Returns: + an argparse.Namespace object containing the parsed values. + Raises: + an error if an argument from unknown_arguments is not a key in config + """ + further_possible_command_line_arguments = [] + for section in config.sections(): + for key in config.options(section): + further_possible_command_line_arguments.append("--" + section + + ":" + key) + for key in config.defaults(): + further_possible_command_line_arguments.append("--DEFAULT:" + key) + + parser = ArgumentParser() + for argument in further_possible_command_line_arguments: + parser.add_argument(argument) + + return parser.parse_args(unknown_arguments) + + +def override_config_with_cli_arguments(config, config_overrides): + arg_dict = vars(config_overrides) + for key in arg_dict: + if arg_dict[key] is not None: + split_key = key.split(":") + if len(split_key) != 2: + raise ValueError("Command line argument must be in the style " + "of --SECTION:key. Please check that there " + "is exactly one colon. You provided: %s" % + key) + config.set(split_key[0], split_key[1], arg_dict[key]) + + return config + + +def save_config_to_file(file, config): + for section in config.sections(): + file.write("[DEFAULT]\n") + for key in config.defaults(): + if config.get(section, key) is None or \ + config.get(section, key) != "": + file.write(key + " = " + config.get("DEFAULT", key) + "\n") + + file.write("[" + section + "]\n") + for key in config.options(section): + if config.get(section, key) is None or \ + config.get(section, key) != "": + file.write(key + " = " + config.get(section, key) + "\n") + def main(): @@ -101,11 +161,17 @@ def main(): comments inside this function and the general HPOlib documentation.""" experiment_dir = os.getcwd() check_before_start._check_zeroth(experiment_dir) - args = use_option_parser() + args, unknown = use_option_parser() optimizer = args.optimizer config_file = os.path.join(experiment_dir, "config.cfg") config = parse_config(config_file, allow_no_value=True, optimizer_module=optimizer) + # Override config values with command line values + config_overrides = parse_config_values_from_unknown_arguments(unknown, + config) + config = override_config_with_cli_arguments(config, config_overrides) + # Saving the config file is down further at the bottom, as soon as we get + # hold of the new optimizer directory # _check_runsolver, _check_modules() check_before_start._check_first(experiment_dir) @@ -129,6 +195,10 @@ def main(): experiment_dir) cmd += optimizer_call + with open(os.path.join(optimizer_dir, "config.cfg"), "w") as f: + config.set("DEFAULT", "is_not_original_config_file", "True") + save_config_to_file(f, config) + # _check_function check_before_start._check_second(experiment_dir, optimizer_dir) From f7f5517c2ccc9a86f05f8d8bf7a64d7bdf85a100 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 29 Jan 2014 11:32:27 +0100 Subject: [PATCH 008/139] move whole library to folder HPOlib reorganize folders to clear the way for a setup.py --- Experiment.py => HPOlib/Experiment.py | 0 Locker.py => HPOlib/Locker.py | 0 {Plotting => HPOlib/Plotting}/doAllPlots.sh | 0 {Plotting => HPOlib/Plotting}/plot.py | 0 {Plotting => HPOlib/Plotting}/plotBoxWhisker.py | 0 {Plotting => HPOlib/Plotting}/plotBranin.py | 0 {Plotting => HPOlib/Plotting}/plotTimeTrace.py | 0 {Plotting => HPOlib/Plotting}/plotTrace.py | 0 {Plotting => HPOlib/Plotting}/plotTraceWithStd.py | 0 {Plotting => HPOlib/Plotting}/plothar6.py | 0 {Plotting => HPOlib/Plotting}/results.sh | 0 {Plotting => HPOlib/Plotting}/statistics.py | 0 __init__.py => HPOlib/__init__.py | 0 benchmark_util.py => HPOlib/benchmark_util.py | 0 {benchmarks => HPOlib/benchmarks}/branin/branin.py | 0 {benchmarks => HPOlib/benchmarks}/branin/config.cfg | 0 {benchmarks => HPOlib/benchmarks}/branin/smac/params.txt | 0 {benchmarks => HPOlib/benchmarks}/branin/spearmint/config.pb | 0 {benchmarks => HPOlib/benchmarks}/branin/tpe/space.py | 0 {benchmarks => HPOlib/benchmarks}/har6/config.cfg | 0 {benchmarks => HPOlib/benchmarks}/har6/har6.py | 0 {benchmarks => HPOlib/benchmarks}/har6/smac/params.txt | 0 {benchmarks => HPOlib/benchmarks}/har6/spearmint/config.pb | 0 {benchmarks => HPOlib/benchmarks}/har6/tpe/space.py | 0 check_before_start.py => HPOlib/check_before_start.py | 0 {config_parser => HPOlib/config_parser}/__init__.py | 0 {config_parser => HPOlib/config_parser}/generalDefault.cfg | 0 {config_parser => HPOlib/config_parser}/parse.py | 0 {config_parser => HPOlib/config_parser}/smacDefault.cfg | 0 {config_parser => HPOlib/config_parser}/smac_parser.py | 0 {config_parser => HPOlib/config_parser}/spearmintDefault.cfg | 0 {config_parser => HPOlib/config_parser}/spearmint_parser.py | 0 {config_parser => HPOlib/config_parser}/tpeDefault.cfg | 0 {config_parser => HPOlib/config_parser}/tpe_parser.py | 0 cv.py => HPOlib/cv.py | 0 data_util.py => HPOlib/data_util.py | 0 {format_converter => HPOlib/format_converter}/Readme.txt | 0 {format_converter => HPOlib/format_converter}/SMACSpearmint.py | 0 {format_converter => HPOlib/format_converter}/TpeSMAC.py | 0 {optimizers => HPOlib/optimizers}/__init__.py | 0 .../optimizers}/hyperopt_august2013_mod/__init__.py | 0 {optimizers => HPOlib/optimizers}/smac_2_06_01-dev/dummy.py | 0 .../optimizers}/spearmint_april2013_mod/__init__.py | 0 randomtpe.py => HPOlib/randomtpe.py | 0 run_instance.py => HPOlib/run_instance.py | 0 run_test.py => HPOlib/run_test.py | 0 {runsolver => HPOlib/runsolver}/dummy.py | 0 runsolver_wrapper.py => HPOlib/runsolver_wrapper.py | 0 smac.py => HPOlib/smac.py | 0 spearmint.py => HPOlib/spearmint.py | 0 tpe.py => HPOlib/tpe.py | 0 tpecall.py => HPOlib/tpecall.py | 0 wrapping.py => HPOlib/wrapping.py | 0 wrapping_util.py => HPOlib/wrapping_util.py | 0 README.md => README.txt | 0 55 files changed, 0 insertions(+), 0 deletions(-) rename Experiment.py => HPOlib/Experiment.py (100%) rename Locker.py => HPOlib/Locker.py (100%) rename {Plotting => HPOlib/Plotting}/doAllPlots.sh (100%) rename {Plotting => HPOlib/Plotting}/plot.py (100%) rename {Plotting => HPOlib/Plotting}/plotBoxWhisker.py (100%) rename {Plotting => HPOlib/Plotting}/plotBranin.py (100%) rename {Plotting => HPOlib/Plotting}/plotTimeTrace.py (100%) rename {Plotting => HPOlib/Plotting}/plotTrace.py (100%) rename {Plotting => HPOlib/Plotting}/plotTraceWithStd.py (100%) rename {Plotting => HPOlib/Plotting}/plothar6.py (100%) rename {Plotting => HPOlib/Plotting}/results.sh (100%) rename {Plotting => HPOlib/Plotting}/statistics.py (100%) rename __init__.py => HPOlib/__init__.py (100%) rename benchmark_util.py => HPOlib/benchmark_util.py (100%) rename {benchmarks => HPOlib/benchmarks}/branin/branin.py (100%) rename {benchmarks => HPOlib/benchmarks}/branin/config.cfg (100%) rename {benchmarks => HPOlib/benchmarks}/branin/smac/params.txt (100%) rename {benchmarks => HPOlib/benchmarks}/branin/spearmint/config.pb (100%) rename {benchmarks => HPOlib/benchmarks}/branin/tpe/space.py (100%) rename {benchmarks => HPOlib/benchmarks}/har6/config.cfg (100%) rename {benchmarks => HPOlib/benchmarks}/har6/har6.py (100%) rename {benchmarks => HPOlib/benchmarks}/har6/smac/params.txt (100%) rename {benchmarks => HPOlib/benchmarks}/har6/spearmint/config.pb (100%) rename {benchmarks => HPOlib/benchmarks}/har6/tpe/space.py (100%) rename check_before_start.py => HPOlib/check_before_start.py (100%) rename {config_parser => HPOlib/config_parser}/__init__.py (100%) rename {config_parser => HPOlib/config_parser}/generalDefault.cfg (100%) rename {config_parser => HPOlib/config_parser}/parse.py (100%) rename {config_parser => HPOlib/config_parser}/smacDefault.cfg (100%) rename {config_parser => HPOlib/config_parser}/smac_parser.py (100%) rename {config_parser => HPOlib/config_parser}/spearmintDefault.cfg (100%) rename {config_parser => HPOlib/config_parser}/spearmint_parser.py (100%) rename {config_parser => HPOlib/config_parser}/tpeDefault.cfg (100%) rename {config_parser => HPOlib/config_parser}/tpe_parser.py (100%) rename cv.py => HPOlib/cv.py (100%) rename data_util.py => HPOlib/data_util.py (100%) rename {format_converter => HPOlib/format_converter}/Readme.txt (100%) rename {format_converter => HPOlib/format_converter}/SMACSpearmint.py (100%) rename {format_converter => HPOlib/format_converter}/TpeSMAC.py (100%) rename {optimizers => HPOlib/optimizers}/__init__.py (100%) rename {optimizers => HPOlib/optimizers}/hyperopt_august2013_mod/__init__.py (100%) rename {optimizers => HPOlib/optimizers}/smac_2_06_01-dev/dummy.py (100%) rename {optimizers => HPOlib/optimizers}/spearmint_april2013_mod/__init__.py (100%) rename randomtpe.py => HPOlib/randomtpe.py (100%) rename run_instance.py => HPOlib/run_instance.py (100%) rename run_test.py => HPOlib/run_test.py (100%) rename {runsolver => HPOlib/runsolver}/dummy.py (100%) rename runsolver_wrapper.py => HPOlib/runsolver_wrapper.py (100%) rename smac.py => HPOlib/smac.py (100%) rename spearmint.py => HPOlib/spearmint.py (100%) rename tpe.py => HPOlib/tpe.py (100%) rename tpecall.py => HPOlib/tpecall.py (100%) rename wrapping.py => HPOlib/wrapping.py (100%) rename wrapping_util.py => HPOlib/wrapping_util.py (100%) rename README.md => README.txt (100%) diff --git a/Experiment.py b/HPOlib/Experiment.py similarity index 100% rename from Experiment.py rename to HPOlib/Experiment.py diff --git a/Locker.py b/HPOlib/Locker.py similarity index 100% rename from Locker.py rename to HPOlib/Locker.py diff --git a/Plotting/doAllPlots.sh b/HPOlib/Plotting/doAllPlots.sh similarity index 100% rename from Plotting/doAllPlots.sh rename to HPOlib/Plotting/doAllPlots.sh diff --git a/Plotting/plot.py b/HPOlib/Plotting/plot.py similarity index 100% rename from Plotting/plot.py rename to HPOlib/Plotting/plot.py diff --git a/Plotting/plotBoxWhisker.py b/HPOlib/Plotting/plotBoxWhisker.py similarity index 100% rename from Plotting/plotBoxWhisker.py rename to HPOlib/Plotting/plotBoxWhisker.py diff --git a/Plotting/plotBranin.py b/HPOlib/Plotting/plotBranin.py similarity index 100% rename from Plotting/plotBranin.py rename to HPOlib/Plotting/plotBranin.py diff --git a/Plotting/plotTimeTrace.py b/HPOlib/Plotting/plotTimeTrace.py similarity index 100% rename from Plotting/plotTimeTrace.py rename to HPOlib/Plotting/plotTimeTrace.py diff --git a/Plotting/plotTrace.py b/HPOlib/Plotting/plotTrace.py similarity index 100% rename from Plotting/plotTrace.py rename to HPOlib/Plotting/plotTrace.py diff --git a/Plotting/plotTraceWithStd.py b/HPOlib/Plotting/plotTraceWithStd.py similarity index 100% rename from Plotting/plotTraceWithStd.py rename to HPOlib/Plotting/plotTraceWithStd.py diff --git a/Plotting/plothar6.py b/HPOlib/Plotting/plothar6.py similarity index 100% rename from Plotting/plothar6.py rename to HPOlib/Plotting/plothar6.py diff --git a/Plotting/results.sh b/HPOlib/Plotting/results.sh similarity index 100% rename from Plotting/results.sh rename to HPOlib/Plotting/results.sh diff --git a/Plotting/statistics.py b/HPOlib/Plotting/statistics.py similarity index 100% rename from Plotting/statistics.py rename to HPOlib/Plotting/statistics.py diff --git a/__init__.py b/HPOlib/__init__.py similarity index 100% rename from __init__.py rename to HPOlib/__init__.py diff --git a/benchmark_util.py b/HPOlib/benchmark_util.py similarity index 100% rename from benchmark_util.py rename to HPOlib/benchmark_util.py diff --git a/benchmarks/branin/branin.py b/HPOlib/benchmarks/branin/branin.py similarity index 100% rename from benchmarks/branin/branin.py rename to HPOlib/benchmarks/branin/branin.py diff --git a/benchmarks/branin/config.cfg b/HPOlib/benchmarks/branin/config.cfg similarity index 100% rename from benchmarks/branin/config.cfg rename to HPOlib/benchmarks/branin/config.cfg diff --git a/benchmarks/branin/smac/params.txt b/HPOlib/benchmarks/branin/smac/params.txt similarity index 100% rename from benchmarks/branin/smac/params.txt rename to HPOlib/benchmarks/branin/smac/params.txt diff --git a/benchmarks/branin/spearmint/config.pb b/HPOlib/benchmarks/branin/spearmint/config.pb similarity index 100% rename from benchmarks/branin/spearmint/config.pb rename to HPOlib/benchmarks/branin/spearmint/config.pb diff --git a/benchmarks/branin/tpe/space.py b/HPOlib/benchmarks/branin/tpe/space.py similarity index 100% rename from benchmarks/branin/tpe/space.py rename to HPOlib/benchmarks/branin/tpe/space.py diff --git a/benchmarks/har6/config.cfg b/HPOlib/benchmarks/har6/config.cfg similarity index 100% rename from benchmarks/har6/config.cfg rename to HPOlib/benchmarks/har6/config.cfg diff --git a/benchmarks/har6/har6.py b/HPOlib/benchmarks/har6/har6.py similarity index 100% rename from benchmarks/har6/har6.py rename to HPOlib/benchmarks/har6/har6.py diff --git a/benchmarks/har6/smac/params.txt b/HPOlib/benchmarks/har6/smac/params.txt similarity index 100% rename from benchmarks/har6/smac/params.txt rename to HPOlib/benchmarks/har6/smac/params.txt diff --git a/benchmarks/har6/spearmint/config.pb b/HPOlib/benchmarks/har6/spearmint/config.pb similarity index 100% rename from benchmarks/har6/spearmint/config.pb rename to HPOlib/benchmarks/har6/spearmint/config.pb diff --git a/benchmarks/har6/tpe/space.py b/HPOlib/benchmarks/har6/tpe/space.py similarity index 100% rename from benchmarks/har6/tpe/space.py rename to HPOlib/benchmarks/har6/tpe/space.py diff --git a/check_before_start.py b/HPOlib/check_before_start.py similarity index 100% rename from check_before_start.py rename to HPOlib/check_before_start.py diff --git a/config_parser/__init__.py b/HPOlib/config_parser/__init__.py similarity index 100% rename from config_parser/__init__.py rename to HPOlib/config_parser/__init__.py diff --git a/config_parser/generalDefault.cfg b/HPOlib/config_parser/generalDefault.cfg similarity index 100% rename from config_parser/generalDefault.cfg rename to HPOlib/config_parser/generalDefault.cfg diff --git a/config_parser/parse.py b/HPOlib/config_parser/parse.py similarity index 100% rename from config_parser/parse.py rename to HPOlib/config_parser/parse.py diff --git a/config_parser/smacDefault.cfg b/HPOlib/config_parser/smacDefault.cfg similarity index 100% rename from config_parser/smacDefault.cfg rename to HPOlib/config_parser/smacDefault.cfg diff --git a/config_parser/smac_parser.py b/HPOlib/config_parser/smac_parser.py similarity index 100% rename from config_parser/smac_parser.py rename to HPOlib/config_parser/smac_parser.py diff --git a/config_parser/spearmintDefault.cfg b/HPOlib/config_parser/spearmintDefault.cfg similarity index 100% rename from config_parser/spearmintDefault.cfg rename to HPOlib/config_parser/spearmintDefault.cfg diff --git a/config_parser/spearmint_parser.py b/HPOlib/config_parser/spearmint_parser.py similarity index 100% rename from config_parser/spearmint_parser.py rename to HPOlib/config_parser/spearmint_parser.py diff --git a/config_parser/tpeDefault.cfg b/HPOlib/config_parser/tpeDefault.cfg similarity index 100% rename from config_parser/tpeDefault.cfg rename to HPOlib/config_parser/tpeDefault.cfg diff --git a/config_parser/tpe_parser.py b/HPOlib/config_parser/tpe_parser.py similarity index 100% rename from config_parser/tpe_parser.py rename to HPOlib/config_parser/tpe_parser.py diff --git a/cv.py b/HPOlib/cv.py similarity index 100% rename from cv.py rename to HPOlib/cv.py diff --git a/data_util.py b/HPOlib/data_util.py similarity index 100% rename from data_util.py rename to HPOlib/data_util.py diff --git a/format_converter/Readme.txt b/HPOlib/format_converter/Readme.txt similarity index 100% rename from format_converter/Readme.txt rename to HPOlib/format_converter/Readme.txt diff --git a/format_converter/SMACSpearmint.py b/HPOlib/format_converter/SMACSpearmint.py similarity index 100% rename from format_converter/SMACSpearmint.py rename to HPOlib/format_converter/SMACSpearmint.py diff --git a/format_converter/TpeSMAC.py b/HPOlib/format_converter/TpeSMAC.py similarity index 100% rename from format_converter/TpeSMAC.py rename to HPOlib/format_converter/TpeSMAC.py diff --git a/optimizers/__init__.py b/HPOlib/optimizers/__init__.py similarity index 100% rename from optimizers/__init__.py rename to HPOlib/optimizers/__init__.py diff --git a/optimizers/hyperopt_august2013_mod/__init__.py b/HPOlib/optimizers/hyperopt_august2013_mod/__init__.py similarity index 100% rename from optimizers/hyperopt_august2013_mod/__init__.py rename to HPOlib/optimizers/hyperopt_august2013_mod/__init__.py diff --git a/optimizers/smac_2_06_01-dev/dummy.py b/HPOlib/optimizers/smac_2_06_01-dev/dummy.py similarity index 100% rename from optimizers/smac_2_06_01-dev/dummy.py rename to HPOlib/optimizers/smac_2_06_01-dev/dummy.py diff --git a/optimizers/spearmint_april2013_mod/__init__.py b/HPOlib/optimizers/spearmint_april2013_mod/__init__.py similarity index 100% rename from optimizers/spearmint_april2013_mod/__init__.py rename to HPOlib/optimizers/spearmint_april2013_mod/__init__.py diff --git a/randomtpe.py b/HPOlib/randomtpe.py similarity index 100% rename from randomtpe.py rename to HPOlib/randomtpe.py diff --git a/run_instance.py b/HPOlib/run_instance.py similarity index 100% rename from run_instance.py rename to HPOlib/run_instance.py diff --git a/run_test.py b/HPOlib/run_test.py similarity index 100% rename from run_test.py rename to HPOlib/run_test.py diff --git a/runsolver/dummy.py b/HPOlib/runsolver/dummy.py similarity index 100% rename from runsolver/dummy.py rename to HPOlib/runsolver/dummy.py diff --git a/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py similarity index 100% rename from runsolver_wrapper.py rename to HPOlib/runsolver_wrapper.py diff --git a/smac.py b/HPOlib/smac.py similarity index 100% rename from smac.py rename to HPOlib/smac.py diff --git a/spearmint.py b/HPOlib/spearmint.py similarity index 100% rename from spearmint.py rename to HPOlib/spearmint.py diff --git a/tpe.py b/HPOlib/tpe.py similarity index 100% rename from tpe.py rename to HPOlib/tpe.py diff --git a/tpecall.py b/HPOlib/tpecall.py similarity index 100% rename from tpecall.py rename to HPOlib/tpecall.py diff --git a/wrapping.py b/HPOlib/wrapping.py similarity index 100% rename from wrapping.py rename to HPOlib/wrapping.py diff --git a/wrapping_util.py b/HPOlib/wrapping_util.py similarity index 100% rename from wrapping_util.py rename to HPOlib/wrapping_util.py diff --git a/README.md b/README.txt similarity index 100% rename from README.md rename to README.txt From f23cc5bac4c4cd6c43c90c9e0e721c51a3522256 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 29 Jan 2014 11:41:38 +0100 Subject: [PATCH 009/139] move runsolver directory to root directory automatically adjust $PATH to find runsolver, when it is installed in HPOlib/runsolver add .idea files (for pycharm) to .gitignore --- .gitignore | 11 ++++++----- HPOlib/check_before_start.py | 3 ++- HPOlib/wrapping.py | 5 ++++- {HPOlib/runsolver => runsolver}/dummy.py | 0 4 files changed, 12 insertions(+), 7 deletions(-) rename {HPOlib/runsolver => runsolver}/dummy.py (100%) diff --git a/.gitignore b/.gitignore index 0210d597..3713f455 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,15 @@ *.py[cod] # Optimizers -optimizers/smac_2_06_01-dev/* -optimizers/spearmint_april2013_mod/* -optimizers/hyperopt_august2013_mod/* +HPOlib/optimizers/smac_2_06_01-dev/* +HPOlib/optimizers/spearmint_april2013_mod/* +HPOlib/optimizers/hyperopt_august2013_mod/* # Runsolver -runsolver/src/* +HPOlib/runsolver/src/* # benchmark runs -benchmarks/*/*_*/* +HPOlib/benchmarks/*/*_*/* # C extensions *.so @@ -45,3 +45,4 @@ nosetests.xml .mr.developer.cfg .project .pydevproject +.idea diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index 142957cc..a5dc74ec 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -36,7 +36,8 @@ def _check_runsolver(): if stdoutdata is not None and "runsolver" in stdoutdata: pass else: - raise Exception("Runsolver cannot not be found. Are you sure that it's installed?") + raise Exception("Runsolver cannot not be found. Are you sure that it's installed?\n" + "Your $PATH is: " + os.environ['PATH']) def _check_modules(): diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index d52bb0a0..347e660b 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -172,12 +172,15 @@ def main(): config = override_config_with_cli_arguments(config, config_overrides) # Saving the config file is down further at the bottom, as soon as we get # hold of the new optimizer directory + wrapping_dir = os.path.dirname(os.path.realpath(__file__)) + + # Try adding runsolver to path + os.putenv('PATH', os.environ['PATH'] + ":" + wrapping_dir + "/../runsolver/src/") # _check_runsolver, _check_modules() check_before_start._check_first(experiment_dir) # build call - wrapping_dir = os.path.dirname(os.path.realpath(__file__)) cmd = "export PYTHONPATH=$PYTHONPATH:" + wrapping_dir + "\n" # Load optimizer diff --git a/HPOlib/runsolver/dummy.py b/runsolver/dummy.py similarity index 100% rename from HPOlib/runsolver/dummy.py rename to runsolver/dummy.py From 3aa01706cfb7735540fa4f193b91907bc3cac0b4 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 29 Jan 2014 16:58:36 +0100 Subject: [PATCH 010/139] makes tpe version explicit tpe related files are now called hyperopt_august2013_modin order to make it clear which version was/is used to run an experiment. This needs an additional argument 'optimizer_version' in the DEFAULT section in *.cfg, which is set by the ${OPTIMIZER}_parser.py and is the same string like the optimizer version, e.g. 'hyperopt_august2013_mod' affected files: HPOlib/config_parser/hyperopt_august2013_mod_parser.py HPOlib/config_parser/hyperopt_august2013_modDefault.cfg HPOlib/hyperopt_august2013_mod.pyHPOlib/config_parser/hyperopt_august2013_mod_parser.py and in every benchmark folder: HPOlib/benchmarks/*/hyperopt_august2013_mod/space.py It is still possible to call wrapping.py -o tpe, because the relevant tpe-files link to hyperopt_august2013_mod-files Add pymongo as an dependency in check_before_start --- .gitignore | 7 +- .../{tpe => hyperopt_august2013_mod}/space.py | 0 HPOlib/benchmarks/branin/tpe | 1 + .../{tpe => hyperopt_august2013_mod}/space.py | 0 HPOlib/benchmarks/har6/tpe | 1 + HPOlib/check_before_start.py | 10 +- .../hyperopt_august2013_modDefault.cfg | 5 + .../hyperopt_august2013_mod_parser.py | 56 ++++++++ HPOlib/config_parser/tpeDefault.cfg | 5 +- HPOlib/config_parser/tpe_parser.py | 52 +------- HPOlib/cv.py | 11 +- HPOlib/hyperopt_august2013_mod.py | 126 ++++++++++++++++++ HPOlib/runsolver_wrapper.py | 33 +++-- HPOlib/tpe.py | 126 +----------------- HPOlib/wrapping.py | 39 ++++-- 15 files changed, 261 insertions(+), 211 deletions(-) rename HPOlib/benchmarks/branin/{tpe => hyperopt_august2013_mod}/space.py (100%) create mode 120000 HPOlib/benchmarks/branin/tpe rename HPOlib/benchmarks/har6/{tpe => hyperopt_august2013_mod}/space.py (100%) create mode 120000 HPOlib/benchmarks/har6/tpe create mode 100644 HPOlib/config_parser/hyperopt_august2013_modDefault.cfg create mode 100644 HPOlib/config_parser/hyperopt_august2013_mod_parser.py mode change 100644 => 120000 HPOlib/config_parser/tpeDefault.cfg mode change 100644 => 120000 HPOlib/config_parser/tpe_parser.py create mode 100644 HPOlib/hyperopt_august2013_mod.py mode change 100644 => 120000 HPOlib/tpe.py diff --git a/.gitignore b/.gitignore index 3713f455..01ff82ea 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ HPOlib/optimizers/spearmint_april2013_mod/* HPOlib/optimizers/hyperopt_august2013_mod/* # Runsolver -HPOlib/runsolver/src/* +runsolver/src/* # benchmark runs HPOlib/benchmarks/*/*_*/* @@ -45,4 +45,9 @@ nosetests.xml .mr.developer.cfg .project .pydevproject + +# pycharm .idea + +# Others +*~ diff --git a/HPOlib/benchmarks/branin/tpe/space.py b/HPOlib/benchmarks/branin/hyperopt_august2013_mod/space.py similarity index 100% rename from HPOlib/benchmarks/branin/tpe/space.py rename to HPOlib/benchmarks/branin/hyperopt_august2013_mod/space.py diff --git a/HPOlib/benchmarks/branin/tpe b/HPOlib/benchmarks/branin/tpe new file mode 120000 index 00000000..398d85cc --- /dev/null +++ b/HPOlib/benchmarks/branin/tpe @@ -0,0 +1 @@ +hyperopt_august2013_mod/ \ No newline at end of file diff --git a/HPOlib/benchmarks/har6/tpe/space.py b/HPOlib/benchmarks/har6/hyperopt_august2013_mod/space.py similarity index 100% rename from HPOlib/benchmarks/har6/tpe/space.py rename to HPOlib/benchmarks/har6/hyperopt_august2013_mod/space.py diff --git a/HPOlib/benchmarks/har6/tpe b/HPOlib/benchmarks/har6/tpe new file mode 120000 index 00000000..398d85cc --- /dev/null +++ b/HPOlib/benchmarks/har6/tpe @@ -0,0 +1 @@ +hyperopt_august2013_mod/ \ No newline at end of file diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index a5dc74ec..fb113091 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -56,14 +56,20 @@ def _check_modules(): % scipy.__version__ except: raise ImportError("Scipy cannot be imported. Are you sure that it's installed?") - #import bson + import networkx import google.protobuf try: import theano except ImportError: - print "Theano not found" + print "Theano not found. You might need this to run some more complex benchmarks!" + + try: + import pymongo + from bson.objectid import ObjectId + except ImportError: + raise ImportError("Pymongo cannot be imported. Are you sure it's installed?") if 'cuda' not in os.environ['PATH']: print "\tCUDA not in $PATH" diff --git a/HPOlib/config_parser/hyperopt_august2013_modDefault.cfg b/HPOlib/config_parser/hyperopt_august2013_modDefault.cfg new file mode 100644 index 00000000..2b578b8f --- /dev/null +++ b/HPOlib/config_parser/hyperopt_august2013_modDefault.cfg @@ -0,0 +1,5 @@ +[TPE] +space = space.py +version = hyperopt_august2013_mod +#exp_dir = %(exp_directory)s/tpe/ +#numberEvals = %(numberOfJobs)s \ No newline at end of file diff --git a/HPOlib/config_parser/hyperopt_august2013_mod_parser.py b/HPOlib/config_parser/hyperopt_august2013_mod_parser.py new file mode 100644 index 00000000..cbd75a80 --- /dev/null +++ b/HPOlib/config_parser/hyperopt_august2013_mod_parser.py @@ -0,0 +1,56 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os + +import ConfigParser + + +def add_default(config): + # This module reads tpeDefault.cfg and adds this defaults to a given config + assert isinstance(config, ConfigParser.RawConfigParser), \ + "config is not a valid instance" + + config_fn = os.path.join(os.path.dirname(os.path.realpath(__file__)), + "tpeDefault.cfg") + if not os.path.isfile(config_fn): + raise Exception('%s is not a valid file\n' % config_fn) + + tpe_config = ConfigParser.SafeConfigParser(allow_no_value=True) + tpe_config.read(config_fn) + # -------------------------------------------------------------------------- + # TPE + # -------------------------------------------------------------------------- + # Set default for TPE + if not config.has_section('TPE'): + config.add_section('TPE') + + # optional cases + if not config.has_option('TPE', 'space'): + config.set('TPE', 'space', + tpe_config.get('TPE', 'space')) + + if not config.has_option('TPE', 'numberEvals'): + config.set('TPE', 'numberEvals', + config.get('DEFAULT', 'numberOfJobs')) + + # Anyway set this + # Makes it possible to call, e.g. hyperopt_august2013_mod via simply tpe + config.set('DEFAULT', 'optimizer_version', tpe_config.get('TPE', 'version')) + + return config \ No newline at end of file diff --git a/HPOlib/config_parser/tpeDefault.cfg b/HPOlib/config_parser/tpeDefault.cfg deleted file mode 100644 index 57565812..00000000 --- a/HPOlib/config_parser/tpeDefault.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[TPE] -space = space.py -#exp_dir = %(exp_directory)s/tpe/ -#numberEvals = %(numberOfJobs)s \ No newline at end of file diff --git a/HPOlib/config_parser/tpeDefault.cfg b/HPOlib/config_parser/tpeDefault.cfg new file mode 120000 index 00000000..a3a72127 --- /dev/null +++ b/HPOlib/config_parser/tpeDefault.cfg @@ -0,0 +1 @@ +hyperopt_august2013_modDefault.cfg \ No newline at end of file diff --git a/HPOlib/config_parser/tpe_parser.py b/HPOlib/config_parser/tpe_parser.py deleted file mode 100644 index 2db6dc25..00000000 --- a/HPOlib/config_parser/tpe_parser.py +++ /dev/null @@ -1,51 +0,0 @@ -## -# wrapping: A program making it easy to use hyperparameter -# optimization software. -# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -import ConfigParser - - -def add_default(config): - # This module reads tpeDefault.cfg and adds this defaults to a given config - assert isinstance(config, ConfigParser.RawConfigParser), \ - "config is not a valid instance" - - config_fn = os.path.join(os.path.dirname(os.path.realpath(__file__)), - "tpeDefault.cfg") - if not os.path.isfile(config_fn): - raise Exception('%s is not a valid file\n' % config_fn) - - tpe_config = ConfigParser.SafeConfigParser(allow_no_value=True) - tpe_config.read(config_fn) - # -------------------------------------------------------------------------- - # TPE - # -------------------------------------------------------------------------- - # Set default for TPE - if not config.has_section('TPE'): - config.add_section('TPE') - - # optional cases - if not config.has_option('TPE', 'space'): - config.set('TPE', 'space', - tpe_config.get('TPE', 'space')) - - if not config.has_option('TPE', 'numberEvals'): - config.set('TPE', 'numberEvals', - config.get('DEFAULT', 'numberOfJobs')) - return config \ No newline at end of file diff --git a/HPOlib/config_parser/tpe_parser.py b/HPOlib/config_parser/tpe_parser.py new file mode 120000 index 00000000..f32f95f7 --- /dev/null +++ b/HPOlib/config_parser/tpe_parser.py @@ -0,0 +1 @@ +hyperopt_august2013_mod_parser.py \ No newline at end of file diff --git a/HPOlib/cv.py b/HPOlib/cv.py index 18d648c7..69e86ce3 100644 --- a/HPOlib/cv.py +++ b/HPOlib/cv.py @@ -37,8 +37,15 @@ __contact__ = "automl.org" +# TODO: This should be in a util function sometime in the future +# Is duplicated in runsolver_wrapper.py + +def get_optimizer(): + return "_".join(os.getcwd().split("/")[-1].split("_")[0:-2]) + + def loadExperimentFile(): - optimizer = os.getcwd().split("/")[-1].split("_")[0] + optimizer = get_optimizer() experiment = Experiment(".", optimizer) return experiment @@ -46,7 +53,7 @@ def loadExperimentFile(): def doCV(params, folds=10): print "Starting Cross validation" sys.stdout.flush() - optimizer = os.getcwd().split("/")[-1].split("_")[0] + optimizer = get_optimizer() cfg = load_experiment_config_file() # Now evaluate $fold times diff --git a/HPOlib/hyperopt_august2013_mod.py b/HPOlib/hyperopt_august2013_mod.py new file mode 100644 index 00000000..4b81d9a5 --- /dev/null +++ b/HPOlib/hyperopt_august2013_mod.py @@ -0,0 +1,126 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#!/usr/bin/env python + +import cPickle +import os +import sys + +import wrapping_util + + +version_info = ("# %76s #\n" % "https://github.com/hyperopt/hyperopt/tree/486aebec8a4170e4781d99bbd6cca09123b12717") +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" + +optimizer_str = "hyperopt_august2013_mod" + +def build_tpe_call(config, options, optimizer_dir): + #For TPE we have to cd to the exp_dir + call = "cd " + optimizer_dir + "\n" + \ + "python " + os.path.dirname(os.path.realpath(__file__)) + \ + "/tpecall.py" + call = ' '.join([call, '-p', config.get('TPE', 'space'), + "-a", config.get('DEFAULT', 'algorithm'), + "-m", config.get('TPE', 'numberEvals'), + "-s", str(options.seed)]) + if options.restore: + call = ' '.join([call, '-r']) + return call + + +#noinspection PyUnusedLocal +def restore(config, optimizer_dir, **kwargs): + """ + Returns the number of restored runs. This is the number of different configs + tested multiplied by the number of crossvalidation folds. + """ + restore_file = os.path.join(optimizer_dir, 'state.pkl') + if not os.path.exists(restore_file): + print "Oups, this should have been checked before" + raise Exception("%s does not exist" % (restore_file,)) + + # Special settings for restoring + fh = open(restore_file) + state = cPickle.load(fh) + fh.close() + complete_runs = 0 + #noinspection PyProtectedMember + tpe_trials = state['trials']._trials + for trial in tpe_trials: + # Assumes that all not valid states states are marked crashed + if trial['state'] == 2: + complete_runs += 1 + restored_runs = complete_runs * config.getint('DEFAULT', 'numberCV') + return restored_runs + + +#noinspection PyUnusedLocal +def main(config, options, experiment_dir, **kwargs): + # config: Loaded .cfg file + # options: Options containing seed, restore_dir, + # experiment_dir: Experiment directory/Benchmark_directory + # **kwargs: Nothing so far + time_string = wrapping_util.get_time_string() + cmd = "" + SYSTEM_WIDE = 0 + AUGUST_2013_MOD = 1 + + try: + import hyperopt + version = SYSTEM_WIDE + except ImportError: + cmd += "export PYTHONPATH=$PYTHONPATH:" + os.path.dirname(os.path.abspath(__file__)) + \ + "/optimizers/hyperopt_august2013_mod\n" + import optimizers.hyperopt_august2013_mod.hyperopt as hyperopt + version = AUGUST_2013_MOD + + path_to_optimizer = os.path.abspath(os.path.dirname(hyperopt.__file__)) + + # Find experiment directory + if options.restore: + if not os.path.exists(options.restore): + raise Exception("The restore directory does not exist") + optimizer_dir = options.restore + else: + optimizer_dir = os.path.join(experiment_dir, optimizer_str + "_" + + str(options.seed) + "_" + + time_string) + + # Build call + cmd += build_tpe_call(config, options, optimizer_dir) + + # Set up experiment directory + if not os.path.exists(optimizer_dir): + os.mkdir(optimizer_dir) + space = config.get('TPE', 'space') + # Copy the hyperopt search space + if not os.path.exists(os.path.join(optimizer_dir, space)): + os.symlink(os.path.join(experiment_dir, optimizer_str, space), + os.path.join(optimizer_dir, space)) + sys.stdout.write("### INFORMATION ################################################################\n") + sys.stdout.write("# You're running %40s #\n" % path_to_optimizer) + if version == SYSTEM_WIDE: + pass + else: + sys.stdout.write("# To reproduce our results you need version 0.0.3.dev, which can be found here:#\n") + sys.stdout.write("%s" % version_info) + sys.stdout.write("# A newer version might be available, but not yet built in. #\n") + sys.stdout.write("################################################################################\n") + return cmd, optimizer_dir diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 7290233a..1bb70e94 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -32,8 +32,14 @@ __contact__ = "automl.org" -def load_experiment_file(): - optimizer = os.getcwd().split("/")[-1].split("_")[0] +# TODO: This should be in a util function sometime in the future +# Is duplicated in cv.py +def get_optimizer(): + return "_".join(os.getcwd().split("/")[-1].split("_")[0:-2]) + + +def load_experiment_file(seed): + optimizer = get_optimizer() experiment = Experiment.Experiment(".", optimizer) return experiment @@ -121,13 +127,17 @@ def main(): if len(sys.argv) < 7: sys.stdout.write(usage) exit(1) - - optimizer = os.getcwd().split("/")[-1].split("_")[0] + + # Then get some information for run_instance + fold = int(sys.argv[1]) + seed = int(sys.argv[5]) + + optimizer = get_optimizer() # This has to be done here for SMAC, since smac does not call cv.py - if optimizer == 'smac': + if 'smac' in optimizer: cv_starttime = time.time() - experiment = load_experiment_file() + experiment = load_experiment_file(seed) experiment.start_cv(cv_starttime) del experiment @@ -140,10 +150,6 @@ def main(): memory_limit = cfg.getint('DEFAULT', 'memory_limit') cpu_limit = cfg.getint('DEFAULT', 'cpu_limit') - # Then get some information for run_instance - fold = int(sys.argv[1]) - seed = int(sys.argv[5]) - # Now build param dict param_list = sys.argv[6:] params = dict() @@ -168,6 +174,7 @@ def main(): "run_instance.py") + \ " --fold %d --config %s --seed %d --pkl %s" % \ (fold, cfg_filename, seed, optimizer + ".pkl") + # Do not write the actual task in quotes because runsolver will not work # then delay = 0 @@ -214,7 +221,7 @@ def main(): result_float = float(result_array[6].strip(",")) if not np.isfinite(result_float): replace_nan_in_last_trial() - experiment = load_experiment_file() + experiment = load_experiment_file(seed) last_experiment = experiment.instance_order[-1] assert last_experiment is not None experiment.instance_results[last_experiment[0]][last_experiment[1]]\ @@ -244,8 +251,8 @@ def main(): #os.remove(run_instance_output) #os.remove(runsolver_output_file) #os.remove(params_filename) - if optimizer == 'smac': - experiment = load_experiment_file() + if 'smac' in optimizer: + experiment = load_experiment_file(seed) experiment.end_cv(time.time()) del experiment return return_string diff --git a/HPOlib/tpe.py b/HPOlib/tpe.py deleted file mode 100644 index 1004754e..00000000 --- a/HPOlib/tpe.py +++ /dev/null @@ -1,125 +0,0 @@ -## -# wrapping: A program making it easy to use hyperparameter -# optimization software. -# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -#!/usr/bin/env python - -import cPickle -import os -import sys - -import wrapping_util - - -version_info = ("# %76s #\n" % "https://github.com/hyperopt/hyperopt/tree/486aebec8a4170e4781d99bbd6cca09123b12717") -__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] -__contact__ = "automl.org" - - -def build_tpe_call(config, options, optimizer_dir): - #For TPE we have to cd to the exp_dir - call = "cd " + optimizer_dir + "\n" + \ - "python " + os.path.dirname(os.path.realpath(__file__)) + \ - "/tpecall.py" - call = ' '.join([call, '-p', config.get('TPE', 'space'), - "-a", config.get('DEFAULT', 'algorithm'), - "-m", config.get('TPE', 'numberEvals'), - "-s", str(options.seed)]) - if options.restore: - call = ' '.join([call, '-r']) - return call - - -#noinspection PyUnusedLocal -def restore(config, optimizer_dir, **kwargs): - """ - Returns the number of restored runs. This is the number of different configs - tested multiplied by the number of crossvalidation folds. - """ - restore_file = os.path.join(optimizer_dir, 'state.pkl') - if not os.path.exists(restore_file): - print "Oups, this should have been checked before" - raise Exception("%s does not exist" % (restore_file,)) - - # Special settings for restoring - fh = open(restore_file) - state = cPickle.load(fh) - fh.close() - complete_runs = 0 - #noinspection PyProtectedMember - tpe_trials = state['trials']._trials - for trial in tpe_trials: - # Assumes that all not valid states states are marked crashed - if trial['state'] == 2: - complete_runs += 1 - restored_runs = complete_runs * config.getint('DEFAULT', 'numberCV') - return restored_runs - - -#noinspection PyUnusedLocal -def main(config, options, experiment_dir, **kwargs): - # config: Loaded .cfg file - # options: Options containing seed, restore_dir, - # experiment_dir: Experiment directory/Benchmark_directory - # **kwargs: Nothing so far - time_string = wrapping_util.get_time_string() - cmd = "" - SYSTEM_WIDE = 0 - AUGUST_2013_MOD = 1 - - try: - import hyperopt - version = SYSTEM_WIDE - except ImportError: - cmd += "export PYTHONPATH=$PYTHONPATH:" + os.path.dirname(os.path.abspath(__file__)) + \ - "/optimizers/hyperopt_august2013_mod\n" - import optimizers.hyperopt_august2013_mod.hyperopt as hyperopt - version = AUGUST_2013_MOD - - path_to_optimizer = os.path.abspath(os.path.dirname(hyperopt.__file__)) - - # Find experiment directory - if options.restore: - if not os.path.exists(options.restore): - raise Exception("The restore directory does not exist") - optimizer_dir = options.restore - else: - optimizer_dir = os.path.join(experiment_dir, "tpe_" + - str(options.seed) + "_" + - time_string) - - # Build call - cmd += build_tpe_call(config, options, optimizer_dir) - - # Set up experiment directory - if not os.path.exists(optimizer_dir): - os.mkdir(optimizer_dir) - space = config.get('TPE', 'space') - # Copy the hyperopt search space - if not os.path.exists(os.path.join(optimizer_dir, space)): - os.symlink(os.path.join(experiment_dir, "tpe", space), - os.path.join(optimizer_dir, space)) - sys.stdout.write("### INFORMATION ################################################################\n") - sys.stdout.write("# You're running %40s #\n" % path_to_optimizer) - if version == SYSTEM_WIDE: - pass - else: - sys.stdout.write("# To reproduce our results you need version 0.0.3.dev, which can be found here:#\n") - sys.stdout.write("%s" % version_info) - sys.stdout.write("# A newer version might be available, but not yet built in. #\n") - sys.stdout.write("################################################################################\n") - return cmd, optimizer_dir diff --git a/HPOlib/tpe.py b/HPOlib/tpe.py new file mode 120000 index 00000000..64428bd1 --- /dev/null +++ b/HPOlib/tpe.py @@ -0,0 +1 @@ +hyperopt_august2013_mod.py \ No newline at end of file diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 347e660b..3f0d4853 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -18,19 +18,19 @@ #!/usr/bin/env python +from argparse import ArgumentParser +from config_parser.parse import parse_config import imp -import numpy as np import os import subprocess import sys import time +import numpy as np + import check_before_start -from config_parser.parse import parse_config import Experiment -import numpy as np -import argparse -from argparse import ArgumentParser + __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" @@ -82,15 +82,15 @@ def use_option_parser(): parser.add_argument("-o", "--optimizer", action="store", type=str, help="Specify the optimizer name.", required=True) parser.add_argument("-p", "--print", action="store_true", dest="printcmd", - default=False, - help="If set print the command instead of executing it") + default=False, + help="If set print the command instead of executing it") parser.add_argument("-s", "--seed", dest="seed", default=1, type=int, - help="Set the seed of the optimizer") + help="Set the seed of the optimizer") parser.add_argument("-t", "--title", dest="title", default=None, - help="A title for the experiment") + help="A title for the experiment") restore_help_string = "Restore the state from a given directory" parser.add_argument("--restore", default=None, dest="restore", - help=restore_help_string) + help=restore_help_string) args, unknown = parser.parse_known_args() return args, unknown @@ -135,6 +135,11 @@ def override_config_with_cli_arguments(config, config_overrides): "of --SECTION:key. Please check that there " "is exactly one colon. You provided: %s" % key) + if split_key[0] == 'DEFAULT' and split_key[1] == "optimizer_version": + raise NotImplementedError("Overwriting the optimizer_version via a cli is a bad idea." + "Try to do something else!\n" + "You were trying to use %s instead of %s" + % (arg_dict[key], config.get("DEFAULT", "optimizer_version"))) config.set(split_key[0], split_key[1], arg_dict[key]) return config @@ -145,17 +150,16 @@ def save_config_to_file(file, config): file.write("[DEFAULT]\n") for key in config.defaults(): if config.get(section, key) is None or \ - config.get(section, key) != "": + config.get(section, key) != "": file.write(key + " = " + config.get("DEFAULT", key) + "\n") file.write("[" + section + "]\n") for key in config.options(section): if config.get(section, key) is None or \ - config.get(section, key) != "": + config.get(section, key) != "": file.write(key + " = " + config.get(section, key) + "\n") - def main(): """Start an optimization of the HPOlib. For documentation see the comments inside this function and the general HPOlib documentation.""" @@ -166,6 +170,14 @@ def main(): config_file = os.path.join(experiment_dir, "config.cfg") config = parse_config(config_file, allow_no_value=True, optimizer_module=optimizer) + + # Check whether we're calling a optimizer version, that links to another + # e.g. calling tpe, but running hyperopt_august2013_mod + if optimizer != config.get('DEFAULT', 'optimizer_version'): + print("[INFO] You called -o %s, but this is only a link to version %s" % + (optimizer, config.get('DEFAULT', 'optimizer_version'))) + optimizer = config.get('DEFAULT', 'optimizer_version') + # Override config values with command line values config_overrides = parse_config_values_from_unknown_arguments(unknown, config) @@ -218,6 +230,7 @@ def main(): title=args.title) trials.optimizer = optimizer + # TODO: We do not have any old runs anymore. DELETE this! if args.restore: # Old versions did store NaNs instead of the worst possible result for # crashed runs in the instance_results. In order to be able to load From 0f02bb50b6994be70a747c59b0c441dd7f983239 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 29 Jan 2014 17:38:37 +0100 Subject: [PATCH 011/139] change spearmint and smac Same changes like described in 3aa01706cfb7735540fa4f193b91907bc3cac0b4 for hyperopt --- HPOlib/benchmarks/branin/smac | 1 + .../{smac => smac_2_06_01-dev}/params.txt | 0 HPOlib/benchmarks/branin/spearmint | 1 + .../config.pb | 0 HPOlib/benchmarks/har6/smac | 1 + .../{smac => smac_2_06_01-dev}/params.txt | 0 HPOlib/benchmarks/har6/spearmint | 1 + .../config.pb | 0 HPOlib/config_parser/smacDefault.cfg | 28 +-- .../config_parser/smac_2_06_01-devDefault.cfg | 29 +++ .../config_parser/smac_2_06_01-dev_parser.py | 71 ++++++ HPOlib/config_parser/smac_parser.py | 68 +---- HPOlib/config_parser/spearmintDefault.cfg | 22 +- .../spearmint_april2013_modDefault.cfg | 23 ++ .../spearmint_april2013_mod_parser.py | 62 +++++ HPOlib/config_parser/spearmint_parser.py | 59 +---- HPOlib/smac.py | 232 +---------------- HPOlib/smac_2_06_01-dev.py | 233 ++++++++++++++++++ HPOlib/spearmint.py | 118 +-------- HPOlib/spearmint_april2013_mod.py | 119 +++++++++ 20 files changed, 547 insertions(+), 521 deletions(-) create mode 120000 HPOlib/benchmarks/branin/smac rename HPOlib/benchmarks/branin/{smac => smac_2_06_01-dev}/params.txt (100%) create mode 120000 HPOlib/benchmarks/branin/spearmint rename HPOlib/benchmarks/branin/{spearmint => spearmint_april2013_mod}/config.pb (100%) create mode 120000 HPOlib/benchmarks/har6/smac rename HPOlib/benchmarks/har6/{smac => smac_2_06_01-dev}/params.txt (100%) create mode 120000 HPOlib/benchmarks/har6/spearmint rename HPOlib/benchmarks/har6/{spearmint => spearmint_april2013_mod}/config.pb (100%) mode change 100644 => 120000 HPOlib/config_parser/smacDefault.cfg create mode 100644 HPOlib/config_parser/smac_2_06_01-devDefault.cfg create mode 100644 HPOlib/config_parser/smac_2_06_01-dev_parser.py mode change 100644 => 120000 HPOlib/config_parser/smac_parser.py mode change 100644 => 120000 HPOlib/config_parser/spearmintDefault.cfg create mode 100644 HPOlib/config_parser/spearmint_april2013_modDefault.cfg create mode 100644 HPOlib/config_parser/spearmint_april2013_mod_parser.py mode change 100644 => 120000 HPOlib/config_parser/spearmint_parser.py mode change 100644 => 120000 HPOlib/smac.py create mode 100644 HPOlib/smac_2_06_01-dev.py mode change 100644 => 120000 HPOlib/spearmint.py create mode 100644 HPOlib/spearmint_april2013_mod.py diff --git a/HPOlib/benchmarks/branin/smac b/HPOlib/benchmarks/branin/smac new file mode 120000 index 00000000..cd8179ec --- /dev/null +++ b/HPOlib/benchmarks/branin/smac @@ -0,0 +1 @@ +smac_2_06_01-dev \ No newline at end of file diff --git a/HPOlib/benchmarks/branin/smac/params.txt b/HPOlib/benchmarks/branin/smac_2_06_01-dev/params.txt similarity index 100% rename from HPOlib/benchmarks/branin/smac/params.txt rename to HPOlib/benchmarks/branin/smac_2_06_01-dev/params.txt diff --git a/HPOlib/benchmarks/branin/spearmint b/HPOlib/benchmarks/branin/spearmint new file mode 120000 index 00000000..525a4844 --- /dev/null +++ b/HPOlib/benchmarks/branin/spearmint @@ -0,0 +1 @@ +spearmint_april2013_mod/ \ No newline at end of file diff --git a/HPOlib/benchmarks/branin/spearmint/config.pb b/HPOlib/benchmarks/branin/spearmint_april2013_mod/config.pb similarity index 100% rename from HPOlib/benchmarks/branin/spearmint/config.pb rename to HPOlib/benchmarks/branin/spearmint_april2013_mod/config.pb diff --git a/HPOlib/benchmarks/har6/smac b/HPOlib/benchmarks/har6/smac new file mode 120000 index 00000000..97e5766b --- /dev/null +++ b/HPOlib/benchmarks/har6/smac @@ -0,0 +1 @@ +smac_2_06_01-dev/ \ No newline at end of file diff --git a/HPOlib/benchmarks/har6/smac/params.txt b/HPOlib/benchmarks/har6/smac_2_06_01-dev/params.txt similarity index 100% rename from HPOlib/benchmarks/har6/smac/params.txt rename to HPOlib/benchmarks/har6/smac_2_06_01-dev/params.txt diff --git a/HPOlib/benchmarks/har6/spearmint b/HPOlib/benchmarks/har6/spearmint new file mode 120000 index 00000000..525a4844 --- /dev/null +++ b/HPOlib/benchmarks/har6/spearmint @@ -0,0 +1 @@ +spearmint_april2013_mod/ \ No newline at end of file diff --git a/HPOlib/benchmarks/har6/spearmint/config.pb b/HPOlib/benchmarks/har6/spearmint_april2013_mod/config.pb similarity index 100% rename from HPOlib/benchmarks/har6/spearmint/config.pb rename to HPOlib/benchmarks/har6/spearmint_april2013_mod/config.pb diff --git a/HPOlib/config_parser/smacDefault.cfg b/HPOlib/config_parser/smacDefault.cfg deleted file mode 100644 index e9830248..00000000 --- a/HPOlib/config_parser/smacDefault.cfg +++ /dev/null @@ -1,27 +0,0 @@ -[SMAC] -# cutoffTime = runsolver_time_limit + 100 sec - -# Set otherwise -# algoExec = %(run_instance)s -# numConcurrentAlgoExecs = %(numberOfConcurrentJobs)s -# experimentDir = %(exp_directory)s/ -# outputDirectory = %(exp_directory)s/smac/output/ -# totalNumRunsLimit = %(numberOfJobs)s * %(cv)s - -numRun = 0 -intraInstanceObj = MEAN -runObj = QUALITY -instanceFile = train.txt -testInstanceFile = test.txt -#execDir = ./ -p = smac/params.pcs -rf_full_tree_bootstrap = False -rf_split_min = 10 - -adaptiveCapping = false -maxIncumbentRuns = 2000 -numIterations = 2147483647 -runtimeLimit = 2147483647 - -deterministic = True -retryTargetAlgorithmRunCount = 0 \ No newline at end of file diff --git a/HPOlib/config_parser/smacDefault.cfg b/HPOlib/config_parser/smacDefault.cfg new file mode 120000 index 00000000..930268c8 --- /dev/null +++ b/HPOlib/config_parser/smacDefault.cfg @@ -0,0 +1 @@ +smac_2_06_01-devDefault.cfg \ No newline at end of file diff --git a/HPOlib/config_parser/smac_2_06_01-devDefault.cfg b/HPOlib/config_parser/smac_2_06_01-devDefault.cfg new file mode 100644 index 00000000..5eaa6630 --- /dev/null +++ b/HPOlib/config_parser/smac_2_06_01-devDefault.cfg @@ -0,0 +1,29 @@ +[SMAC] +# cutoffTime = runsolver_time_limit + 100 sec + +# Set otherwise +# algoExec = %(run_instance)s +# numConcurrentAlgoExecs = %(numberOfConcurrentJobs)s +# experimentDir = %(exp_directory)s/ +# outputDirectory = %(exp_directory)s/smac/output/ +# totalNumRunsLimit = %(numberOfJobs)s * %(cv)s + +numRun = 0 +intraInstanceObj = MEAN +runObj = QUALITY +instanceFile = train.txt +testInstanceFile = test.txt +#execDir = ./ +p = smac/params.pcs +rf_full_tree_bootstrap = False +rf_split_min = 10 + +adaptiveCapping = false +maxIncumbentRuns = 2000 +numIterations = 2147483647 +runtimeLimit = 2147483647 + +deterministic = True +retryTargetAlgorithmRunCount = 0 + +version = smac_2_06_01-dev \ No newline at end of file diff --git a/HPOlib/config_parser/smac_2_06_01-dev_parser.py b/HPOlib/config_parser/smac_2_06_01-dev_parser.py new file mode 100644 index 00000000..7caaf6d7 --- /dev/null +++ b/HPOlib/config_parser/smac_2_06_01-dev_parser.py @@ -0,0 +1,71 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os + +import ConfigParser + + +def add_default(config): + # This module reads smacDefault.cfg and adds this defaults to a given config + + assert isinstance(config, ConfigParser.RawConfigParser), \ + 'config is not a valid instance' + + config_fn = os.path.join(os.path.dirname(os.path.realpath(__file__)), + "smacDefault.cfg") + if not os.path.isfile(config_fn): + raise Exception('%s is not a valid file\n' % config_fn) + + smac_config = ConfigParser.SafeConfigParser(allow_no_value=True) + smac_config.read(config_fn) + # -------------------------------------------------------------------------- + # SMAC + # -------------------------------------------------------------------------- + # Set defaults for SMAC + if not config.has_section('SMAC'): + config.add_section('SMAC') + # optional arguments (exec_dir taken out as is does not seem to be used) + for option in ('numRun', 'instanceFile', 'intraInstanceObj', 'runObj', + 'testInstanceFile', 'p', 'rf_full_tree_bootstrap', + 'rf_split_min', 'adaptiveCapping', 'maxIncumbentRuns', + 'numIterations', 'runtimeLimit', 'deterministic', + 'retryTargetAlgorithmRunCount'): + if not config.has_option('SMAC', option): + config.set('SMAC', option, + smac_config.get('SMAC', option)) + + # special cases + if not config.has_option('SMAC', 'cutoffTime'): + config.set('SMAC', 'cutoffTime', + str(config.getint('DEFAULT', 'runsolver_time_limit') + 100)) + if not config.has_option('SMAC', 'algoExec'): + config.set('SMAC', 'algoExec', + config.get('DEFAULT', 'run_instance')) + if not config.has_option('SMAC', 'totalNumRunsLimit'): + config.set('SMAC', 'totalNumRunsLimit', + str(config.getint('DEFAULT', 'numberOfJobs') * + config.getint('DEFAULT', 'numberCV'))) + if not config.has_option('SMAC', 'numConcurrentAlgoExecs'): + config.set('SMAC', 'numConcurrentAlgoExecs', + config.get('DEFAULT', 'numberOfConcurrentJobs')) + + # Anyway set this + # Makes it possible to call, e.g. smac_2_06_01-dev via smac + config.set('DEFAULT', 'optimizer_version', smac_config.get('SMAC', 'version')) + return config \ No newline at end of file diff --git a/HPOlib/config_parser/smac_parser.py b/HPOlib/config_parser/smac_parser.py deleted file mode 100644 index 8f5e4054..00000000 --- a/HPOlib/config_parser/smac_parser.py +++ /dev/null @@ -1,67 +0,0 @@ -## -# wrapping: A program making it easy to use hyperparameter -# optimization software. -# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -import ConfigParser - - -def add_default(config): - # This module reads smacDefault.cfg and adds this defaults to a given config - - assert isinstance(config, ConfigParser.RawConfigParser), \ - 'config is not a valid instance' - - config_fn = os.path.join(os.path.dirname(os.path.realpath(__file__)), - "smacDefault.cfg") - if not os.path.isfile(config_fn): - raise Exception('%s is not a valid file\n' % config_fn) - - smac_config = ConfigParser.SafeConfigParser(allow_no_value=True) - smac_config.read(config_fn) - # -------------------------------------------------------------------------- - # SMAC - # -------------------------------------------------------------------------- - # Set defaults for SMAC - if not config.has_section('SMAC'): - config.add_section('SMAC') - # optional arguments (exec_dir taken out as is does not seem to be used) - for option in ('numRun', 'instanceFile', 'intraInstanceObj', 'runObj', - 'testInstanceFile', 'p', 'rf_full_tree_bootstrap', - 'rf_split_min', 'adaptiveCapping', 'maxIncumbentRuns', - 'numIterations', 'runtimeLimit', 'deterministic', - 'retryTargetAlgorithmRunCount'): - if not config.has_option('SMAC', option): - config.set('SMAC', option, - smac_config.get('SMAC', option)) - - # special cases - if not config.has_option('SMAC', 'cutoffTime'): - config.set('SMAC', 'cutoffTime', - str(config.getint('DEFAULT', 'runsolver_time_limit') + 100)) - if not config.has_option('SMAC', 'algoExec'): - config.set('SMAC', 'algoExec', - config.get('DEFAULT', 'run_instance')) - if not config.has_option('SMAC', 'totalNumRunsLimit'): - config.set('SMAC', 'totalNumRunsLimit', - str(config.getint('DEFAULT', 'numberOfJobs') * - config.getint('DEFAULT', 'numberCV'))) - if not config.has_option('SMAC', 'numConcurrentAlgoExecs'): - config.set('SMAC', 'numConcurrentAlgoExecs', - config.get('DEFAULT', 'numberOfConcurrentJobs')) - return config \ No newline at end of file diff --git a/HPOlib/config_parser/smac_parser.py b/HPOlib/config_parser/smac_parser.py new file mode 120000 index 00000000..42644afb --- /dev/null +++ b/HPOlib/config_parser/smac_parser.py @@ -0,0 +1 @@ +smac_2_06_01-dev_parser.py \ No newline at end of file diff --git a/HPOlib/config_parser/spearmintDefault.cfg b/HPOlib/config_parser/spearmintDefault.cfg deleted file mode 100644 index 659d5807..00000000 --- a/HPOlib/config_parser/spearmintDefault.cfg +++ /dev/null @@ -1,21 +0,0 @@ -[SPEARMINT] -#Which spearmint script to call -script = python spearmint_sync.py -# Which chooser to take (string) -method = GPEIOptChooser -#Arguments to pass to chooser module -method_args = -#Number of experiments in initial grid -grid_size = 20000 -#The seed used to initialize initial grid (int) -grid_seed = 1 -#Does not need to be specified, when experiment directory with a 'config.pb' is -#present. For Syntax see 'config.bp' in braninpy directory -config = config.pb -#Run in job_wrapper mode (bool) -wrapper = 0 -#The time in_between successive polls for results (float) -spearmint_polling_time = 3.0 -#General Parameters -max_concurrent = 1 -#max_finished_jobs = %(numberOfJobs)s \ No newline at end of file diff --git a/HPOlib/config_parser/spearmintDefault.cfg b/HPOlib/config_parser/spearmintDefault.cfg new file mode 120000 index 00000000..aa1cdf36 --- /dev/null +++ b/HPOlib/config_parser/spearmintDefault.cfg @@ -0,0 +1 @@ +spearmint_april2013_modDefault.cfg \ No newline at end of file diff --git a/HPOlib/config_parser/spearmint_april2013_modDefault.cfg b/HPOlib/config_parser/spearmint_april2013_modDefault.cfg new file mode 100644 index 00000000..7c5c636b --- /dev/null +++ b/HPOlib/config_parser/spearmint_april2013_modDefault.cfg @@ -0,0 +1,23 @@ +[SPEARMINT] +#Which spearmint script to call +script = python spearmint_sync.py +# Which chooser to take (string) +method = GPEIOptChooser +#Arguments to pass to chooser module +method_args = +#Number of experiments in initial grid +grid_size = 20000 +#The seed used to initialize initial grid (int) +grid_seed = 1 +#Does not need to be specified, when experiment directory with a 'config.pb' is +#present. For Syntax see 'config.bp' in braninpy directory +config = config.pb +#Run in job_wrapper mode (bool) +wrapper = 0 +#The time in_between successive polls for results (float) +spearmint_polling_time = 3.0 +#General Parameters +max_concurrent = 1 +#max_finished_jobs = %(numberOfJobs)s + +version = spearmint_april2013_mod \ No newline at end of file diff --git a/HPOlib/config_parser/spearmint_april2013_mod_parser.py b/HPOlib/config_parser/spearmint_april2013_mod_parser.py new file mode 100644 index 00000000..d34cd277 --- /dev/null +++ b/HPOlib/config_parser/spearmint_april2013_mod_parser.py @@ -0,0 +1,62 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os + +import ConfigParser + + +def add_default(config): + # This module reads spearmintDefault.cfg and adds this defaults to a given config + assert isinstance(config, ConfigParser.RawConfigParser), "config is not a valid instance" + + config_fn = os.path.join(os.path.dirname(os.path.realpath(__file__)), "spearmintDefault.cfg") + if not os.path.isfile(config_fn): + raise Exception('%s is not a valid file\n' % config_fn) + + spearmint_config = ConfigParser.SafeConfigParser(allow_no_value=True) + spearmint_config.read(config_fn) + # -------------------------------------------------------------------------- + # SPEARMINT + # -------------------------------------------------------------------------- + # Set default for SPEARMINT + if not config.has_section('SPEARMINT'): + config.add_section('SPEARMINT') + # optional arguments + for option in ('script', 'method', 'grid_size', 'config', 'grid_seed', + 'spearmint_polling_time', 'max_concurrent', 'method_args'): + if not config.has_option('SPEARMINT', option): + config.set('SPEARMINT', option, + spearmint_config.get('SPEARMINT', option)) + + # special cases + if not config.has_option('SPEARMINT', 'method'): + config.set('SPEARMINT', 'method', + spearmint_config.get('SPEARMINT', 'method')) + config.set('SPEARMINT', 'method-args', + spearmint_config.get('SPEARMINT', 'method-args')) + + # GENERAL + if not config.has_option('SPEARMINT', 'max_finished_jobs'): + config.set('SPEARMINT', 'max_finished_jobs', + config.get('DEFAULT', 'numberOfJobs')) + + # Anyway set this + # Makes it possible to call, e.g. smac_2_06_01-dev via smac + config.set('DEFAULT', 'optimizer_version', spearmint_config.get('SPEARMINT', 'version')) + return config \ No newline at end of file diff --git a/HPOlib/config_parser/spearmint_parser.py b/HPOlib/config_parser/spearmint_parser.py deleted file mode 100644 index 44e2fca6..00000000 --- a/HPOlib/config_parser/spearmint_parser.py +++ /dev/null @@ -1,58 +0,0 @@ -## -# wrapping: A program making it easy to use hyperparameter -# optimization software. -# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -import ConfigParser - - -def add_default(config): - # This module reads spearmintDefault.cfg and adds this defaults to a given config - assert isinstance(config, ConfigParser.RawConfigParser), "config is not a valid instance" - - config_fn = os.path.join(os.path.dirname(os.path.realpath(__file__)), "spearmintDefault.cfg") - if not os.path.isfile(config_fn): - raise Exception('%s is not a valid file\n' % config_fn) - - spearmint_config = ConfigParser.SafeConfigParser(allow_no_value=True) - spearmint_config.read(config_fn) - # -------------------------------------------------------------------------- - # SPEARMINT - # -------------------------------------------------------------------------- - # Set default for SPEARMINT - if not config.has_section('SPEARMINT'): - config.add_section('SPEARMINT') - # optional arguments - for option in ('script', 'method', 'grid_size', 'config', 'grid_seed', - 'spearmint_polling_time', 'max_concurrent', 'method_args'): - if not config.has_option('SPEARMINT', option): - config.set('SPEARMINT', option, - spearmint_config.get('SPEARMINT', option)) - - # special cases - if not config.has_option('SPEARMINT', 'method'): - config.set('SPEARMINT', 'method', - spearmint_config.get('SPEARMINT', 'method')) - config.set('SPEARMINT', 'method-args', - spearmint_config.get('SPEARMINT', 'method-args')) - - # GENERAL - if not config.has_option('SPEARMINT', 'max_finished_jobs'): - config.set('SPEARMINT', 'max_finished_jobs', - config.get('DEFAULT', 'numberOfJobs')) - return config \ No newline at end of file diff --git a/HPOlib/config_parser/spearmint_parser.py b/HPOlib/config_parser/spearmint_parser.py new file mode 120000 index 00000000..a6032ea6 --- /dev/null +++ b/HPOlib/config_parser/spearmint_parser.py @@ -0,0 +1 @@ +spearmint_april2013_mod_parser.py \ No newline at end of file diff --git a/HPOlib/smac.py b/HPOlib/smac.py deleted file mode 100644 index fdac42d7..00000000 --- a/HPOlib/smac.py +++ /dev/null @@ -1,231 +0,0 @@ -## -# wrapping: A program making it easy to use hyperparameter -# optimization software. -# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -#!/usr/bin/env python - -import glob -import os -import re -import subprocess -import sys - -import numpy as np - -import wrapping_util - -__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] -__contact__ = "automl.org" - -path_to_optimizer = "optimizers/smac_2_06_01-dev" -version_info = ("# %76s #\n# %76s #\n# %76s #\n" % - ("Automatic Configurator Library ==> v2.06.01-development-643 (a1f71813a262)", - "Random Forest Library ==> v1.05.01-development-95 (4a8077e95b21)", - "SMAC ==> v2.06.01-development-620 (9380d2c6bab9)")) - - -def _get_state_run(optimizer_dir): - rungroups = glob.glob(optimizer_dir + "scenario-SMAC*") - if len(rungroups) == 1: - rungroup = rungroups[0] - else: - print "Found multiple rungroups, take the newest one." - creation_times = [] - for i, filename in enumerate(rungroups): - creation_times.append(float(os.path.getctime(filename))) - newest = np.argmax(creation_times) - rungroup = rungroups[newest] - print creation_times, newest, rungroup - state_runs = glob.glob(rungroup + "/state-run*") - - if len(state_runs) != 1: - raise Exception("wrapping.py can only restore runs with only one" + - " state-run. Please delete all others you don't want" + - "to use.") - return state_runs[0] - - -def build_smac_call(config, options, optimizer_dir): - call = os.path.dirname(os.path.realpath(__file__)) + "/" + path_to_optimizer + "/smac" - call = " ".join([call, '--numRun', str(options.seed), - '--scenario-file', os.path.join(optimizer_dir, 'scenario.txt'), - '--cutoffTime', config.get('SMAC', 'cutoffTime'), - # The instance file does interfere with state restoration, it will only - # be loaded if no state is restored (look further down in the code - # '--instanceFile', config.get('SMAC', 'instanceFile'), - '--intraInstanceObj', config.get('SMAC', 'intraInstanceObj'), - '--runObj', config.get('SMAC', 'runObj'), - # '--testInstanceFile', config.get('SMAC', 'testInstanceFile'), - '--algoExec', '"python', os.path.dirname(os.path.realpath(__file__)) + "/" + - config.get('SMAC', 'algoExec') + '"', - '--execDir', optimizer_dir, - '-p', config.get('SMAC', 'p'), - # The experiment dir MUST not be specified when restarting, it is set - # further down in the code - # '--experimentDir', optimizer_dir, - '--numIterations', config.get('SMAC', 'numIterations'), - '--totalNumRunsLimit', config.get('SMAC', 'totalNumRunsLimit'), - '--outputDirectory', optimizer_dir, - '--numConcurrentAlgoExecs', config.get('SMAC', 'numConcurrentAlgoExecs'), - # '--runGroupName', config.get('SMAC', 'runGroupName'), - '--maxIncumbentRuns', config.get('SMAC', 'maxIncumbentRuns'), - '--retryTargetAlgorithmRunCount', - config.get('SMAC', 'retryTargetAlgorithmRunCount'), - '--save-runs-every-iteration true', - '--rf-split-min', config.get('SMAC', 'rf_split_min')]) - - if config.getboolean('SMAC', 'deterministic'): - call = " ".join([call, '--deterministic true']) - - if config.getboolean('SMAC', 'adaptiveCapping') and \ - config.get('SMAC', 'runObj') == "RUNTIME": - call = " ".join([call, '--adaptiveCapping true']) - - if config.getboolean('SMAC', 'rf_full_tree_bootstrap'): - call = " ".join([call, '--rf-full-tree-bootstrap true']) - - if options.restore: - state_run = _get_state_run(optimizer_dir) - - restore_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), - os.getcwd(), state_run) - call = " ".join([call, "--restore-scenario", restore_path]) - else: - call = " ".join([call, '--instanceFile', - os.path.join(optimizer_dir, 'train.txt'), - '--testInstanceFile', - os.path.join(optimizer_dir, 'test.txt')]) - return call - - -def restore(config, optimizer_dir, **kwargs): - """ - Returns the number of restored runs. - """ - - ############################################################################ - # Run SMAC in a manner that it restores the files but then exits - fh = open(optimizer_dir + "smac_restart.out", "w") - smac_cmd = re.sub('python ' + os.path.dirname(os.path.realpath(__file__)) + - "/" + config.get('SMAC', 'algoExec'), 'pwd', - kwargs['cmd']) - smac_cmd = re.sub('--outputDirectory ' + optimizer_dir, '--outputDirectory ' - + optimizer_dir + "restart_rungroups", smac_cmd) - print smac_cmd - process = subprocess.Popen(smac_cmd, stdout=fh, stderr=fh, shell=True, - executable="/bin/bash") - print "-----------------------RUNNING----------------------------------" - ret = process.wait() - fh.close() - print "Finished with return code: " + str(ret) - # os.remove("smac_restart.out") - - # read smac.out and look how many states are restored - fh = open(optimizer_dir + "smac_restart.out") - prog = re.compile(r"(Restored) ([0-9]{1,100}) (runs)") - restored_runs = 0 - for line in fh.readlines(): - match = prog.search(line) - if match: - # print "###########Match", match.group(0), match.group(2) - restored_runs = int(match.group(2)) - - # Find out all rungroups and state-runs - ############################################################################ - state_run = _get_state_run(optimizer_dir) - - state_run_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), - os.getcwd(), state_run) - state_runs = glob.glob(state_run_path + "/runs_and_results-it*.csv") - # print state_run_path + "/runs_and_results-it*.csv" - # print state_runs - state_run_iterations = [] - for state_run in state_runs: - match = re.search(r"(runs_and_results-it)([0-9]{1,100})(.csv)", - state_run) - # print state_run - if match: - state_run_iterations.append(float(match.group(2))) - # print state_run_iterations - run_and_results_fn = state_runs[np.argmax(state_run_iterations)] - # print run_and_results_fn - - runs_and_results = open(run_and_results_fn) - lines = runs_and_results.readlines() - state_run_iters = len(lines) - 1 - runs_and_results.close() - - fh.close() - - # TODO: Wait for a fix in SMAC - # In SMAC, right now the number of restored iterations is at least one too high - assert state_run_iters == restored_runs - 1, (state_run_iters, restored_runs) - restored_runs = state_run_iters - - return restored_runs - - -#noinspection PyUnusedLocal -def main(config, options, experiment_dir, **kwargs): - # config: Loaded .cfg file - # options: Options containing seed, restore_dir, - # experiment_dir: Experiment directory/Benchmark_directory - # **kwargs: Nothing so far - time_string = wrapping_util.get_time_string() - - # Find experiment directory - if options.restore: - if not os.path.exists(options.restore): - raise Exception("The restore directory does not exist") - optimizer_dir = options.restore - else: - optimizer_dir = os.path.join(experiment_dir, "smac_" + - str(options.seed) + "_" + time_string) - # Build call - cmd = build_smac_call(config, options, optimizer_dir) - - # Set up experiment directory - if not os.path.exists(optimizer_dir): - os.mkdir(optimizer_dir) - # TODO: This can cause huge problems when the files are located - # somewhere else? - params = os.path.split(config.get('SMAC', "p"))[1] - - # Copy the smac search space and create the instance information - fh = open(os.path.join(optimizer_dir, 'train.txt'), "w") - for i in range(config.getint('DEFAULT', 'numberCV')): - fh.write(str(i) + "\n") - fh.close() - - fh = open(os.path.join(optimizer_dir, 'test.txt'), "w") - for i in range(config.getint('DEFAULT', 'numberCV')): - fh.write(str(i) + "\n") - fh.close() - - fh = open(os.path.join(optimizer_dir, "scenario.txt"), "w") - fh.close() - - if not os.path.exists(os.path.join(optimizer_dir, params)): - os.symlink(os.path.join(experiment_dir, "smac", params), - os.path.join(optimizer_dir, params)) - sys.stdout.write("### INFORMATION ################################################################\n") - sys.stdout.write("# You're running %40s #\n" % path_to_optimizer) - sys.stdout.write("%s" % version_info) - sys.stdout.write("# A newer version might be available, but not yet built in. #\n") - sys.stdout.write("# Please use this version only to reproduce our results on automl.org #\n") - sys.stdout.write("################################################################################\n") - return cmd, optimizer_dir \ No newline at end of file diff --git a/HPOlib/smac.py b/HPOlib/smac.py new file mode 120000 index 00000000..6b7db85c --- /dev/null +++ b/HPOlib/smac.py @@ -0,0 +1 @@ +smac_2_06_01-dev.py \ No newline at end of file diff --git a/HPOlib/smac_2_06_01-dev.py b/HPOlib/smac_2_06_01-dev.py new file mode 100644 index 00000000..0ad317e5 --- /dev/null +++ b/HPOlib/smac_2_06_01-dev.py @@ -0,0 +1,233 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#!/usr/bin/env python + +import glob +import os +import re +import subprocess +import sys + +import numpy as np + +import wrapping_util + +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" + +path_to_optimizer = "optimizers/smac_2_06_01-dev" +version_info = ("# %76s #\n# %76s #\n# %76s #\n" % + ("Automatic Configurator Library ==> v2.06.01-development-643 (a1f71813a262)", + "Random Forest Library ==> v1.05.01-development-95 (4a8077e95b21)", + "SMAC ==> v2.06.01-development-620 (9380d2c6bab9)")) + +optimizer_str = "smac_2_06_01-dev" + + +def _get_state_run(optimizer_dir): + rungroups = glob.glob(optimizer_dir + "scenario-SMAC*") + if len(rungroups) == 1: + rungroup = rungroups[0] + else: + print "Found multiple rungroups, take the newest one." + creation_times = [] + for i, filename in enumerate(rungroups): + creation_times.append(float(os.path.getctime(filename))) + newest = np.argmax(creation_times) + rungroup = rungroups[newest] + print creation_times, newest, rungroup + state_runs = glob.glob(rungroup + "/state-run*") + + if len(state_runs) != 1: + raise Exception("wrapping.py can only restore runs with only one" + + " state-run. Please delete all others you don't want" + + "to use.") + return state_runs[0] + + +def build_smac_call(config, options, optimizer_dir): + call = os.path.dirname(os.path.realpath(__file__)) + "/" + path_to_optimizer + "/smac" + call = " ".join([call, '--numRun', str(options.seed), + '--scenario-file', os.path.join(optimizer_dir, 'scenario.txt'), + '--cutoffTime', config.get('SMAC', 'cutoffTime'), + # The instance file does interfere with state restoration, it will only + # be loaded if no state is restored (look further down in the code + # '--instanceFile', config.get('SMAC', 'instanceFile'), + '--intraInstanceObj', config.get('SMAC', 'intraInstanceObj'), + '--runObj', config.get('SMAC', 'runObj'), + # '--testInstanceFile', config.get('SMAC', 'testInstanceFile'), + '--algoExec', '"python', os.path.dirname(os.path.realpath(__file__)) + "/" + + config.get('SMAC', 'algoExec') + '"', + '--execDir', optimizer_dir, + '-p', config.get('SMAC', 'p'), + # The experiment dir MUST not be specified when restarting, it is set + # further down in the code + # '--experimentDir', optimizer_dir, + '--numIterations', config.get('SMAC', 'numIterations'), + '--totalNumRunsLimit', config.get('SMAC', 'totalNumRunsLimit'), + '--outputDirectory', optimizer_dir, + '--numConcurrentAlgoExecs', config.get('SMAC', 'numConcurrentAlgoExecs'), + # '--runGroupName', config.get('SMAC', 'runGroupName'), + '--maxIncumbentRuns', config.get('SMAC', 'maxIncumbentRuns'), + '--retryTargetAlgorithmRunCount', + config.get('SMAC', 'retryTargetAlgorithmRunCount'), + '--save-runs-every-iteration true', + '--rf-split-min', config.get('SMAC', 'rf_split_min')]) + + if config.getboolean('SMAC', 'deterministic'): + call = " ".join([call, '--deterministic true']) + + if config.getboolean('SMAC', 'adaptiveCapping') and \ + config.get('SMAC', 'runObj') == "RUNTIME": + call = " ".join([call, '--adaptiveCapping true']) + + if config.getboolean('SMAC', 'rf_full_tree_bootstrap'): + call = " ".join([call, '--rf-full-tree-bootstrap true']) + + if options.restore: + state_run = _get_state_run(optimizer_dir) + + restore_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), + os.getcwd(), state_run) + call = " ".join([call, "--restore-scenario", restore_path]) + else: + call = " ".join([call, '--instanceFile', + os.path.join(optimizer_dir, 'train.txt'), + '--testInstanceFile', + os.path.join(optimizer_dir, 'test.txt')]) + return call + + +def restore(config, optimizer_dir, **kwargs): + """ + Returns the number of restored runs. + """ + + ############################################################################ + # Run SMAC in a manner that it restores the files but then exits + fh = open(optimizer_dir + "smac_restart.out", "w") + smac_cmd = re.sub('python ' + os.path.dirname(os.path.realpath(__file__)) + + "/" + config.get('SMAC', 'algoExec'), 'pwd', + kwargs['cmd']) + smac_cmd = re.sub('--outputDirectory ' + optimizer_dir, '--outputDirectory ' + + optimizer_dir + "restart_rungroups", smac_cmd) + print smac_cmd + process = subprocess.Popen(smac_cmd, stdout=fh, stderr=fh, shell=True, + executable="/bin/bash") + print "-----------------------RUNNING----------------------------------" + ret = process.wait() + fh.close() + print "Finished with return code: " + str(ret) + # os.remove("smac_restart.out") + + # read smac.out and look how many states are restored + fh = open(optimizer_dir + "smac_restart.out") + prog = re.compile(r"(Restored) ([0-9]{1,100}) (runs)") + restored_runs = 0 + for line in fh.readlines(): + match = prog.search(line) + if match: + # print "###########Match", match.group(0), match.group(2) + restored_runs = int(match.group(2)) + + # Find out all rungroups and state-runs + ############################################################################ + state_run = _get_state_run(optimizer_dir) + + state_run_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), + os.getcwd(), state_run) + state_runs = glob.glob(state_run_path + "/runs_and_results-it*.csv") + # print state_run_path + "/runs_and_results-it*.csv" + # print state_runs + state_run_iterations = [] + for state_run in state_runs: + match = re.search(r"(runs_and_results-it)([0-9]{1,100})(.csv)", + state_run) + # print state_run + if match: + state_run_iterations.append(float(match.group(2))) + # print state_run_iterations + run_and_results_fn = state_runs[np.argmax(state_run_iterations)] + # print run_and_results_fn + + runs_and_results = open(run_and_results_fn) + lines = runs_and_results.readlines() + state_run_iters = len(lines) - 1 + runs_and_results.close() + + fh.close() + + # TODO: Wait for a fix in SMAC + # In SMAC, right now the number of restored iterations is at least one too high + assert state_run_iters == restored_runs - 1, (state_run_iters, restored_runs) + restored_runs = state_run_iters + + return restored_runs + + +#noinspection PyUnusedLocal +def main(config, options, experiment_dir, **kwargs): + # config: Loaded .cfg file + # options: Options containing seed, restore_dir, + # experiment_dir: Experiment directory/Benchmark_directory + # **kwargs: Nothing so far + time_string = wrapping_util.get_time_string() + + # Find experiment directory + if options.restore: + if not os.path.exists(options.restore): + raise Exception("The restore directory does not exist") + optimizer_dir = options.restore + else: + optimizer_dir = os.path.join(experiment_dir, optimizer_str + "_" + + str(options.seed) + "_" + time_string) + # Build call + cmd = build_smac_call(config, options, optimizer_dir) + + # Set up experiment directory + if not os.path.exists(optimizer_dir): + os.mkdir(optimizer_dir) + # TODO: This can cause huge problems when the files are located + # somewhere else? + params = os.path.split(config.get('SMAC', "p"))[1] + + # Copy the smac search space and create the instance information + fh = open(os.path.join(optimizer_dir, 'train.txt'), "w") + for i in range(config.getint('DEFAULT', 'numberCV')): + fh.write(str(i) + "\n") + fh.close() + + fh = open(os.path.join(optimizer_dir, 'test.txt'), "w") + for i in range(config.getint('DEFAULT', 'numberCV')): + fh.write(str(i) + "\n") + fh.close() + + fh = open(os.path.join(optimizer_dir, "scenario.txt"), "w") + fh.close() + + if not os.path.exists(os.path.join(optimizer_dir, params)): + os.symlink(os.path.join(experiment_dir, optimizer_str, params), + os.path.join(optimizer_dir, params)) + sys.stdout.write("### INFORMATION ################################################################\n") + sys.stdout.write("# You're running %40s #\n" % path_to_optimizer) + sys.stdout.write("%s" % version_info) + sys.stdout.write("# A newer version might be available, but not yet built in. #\n") + sys.stdout.write("# Please use this version only to reproduce our results on automl.org #\n") + sys.stdout.write("################################################################################\n") + return cmd, optimizer_dir \ No newline at end of file diff --git a/HPOlib/spearmint.py b/HPOlib/spearmint.py deleted file mode 100644 index 8d739ba5..00000000 --- a/HPOlib/spearmint.py +++ /dev/null @@ -1,117 +0,0 @@ -## -# wrapping: A program making it easy to use hyperparameter -# optimization software. -# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -#!/usr/bin/env python - -import cPickle -import os -import sys - -import numpy as np - -import wrapping_util - -__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] -__contact__ = "automl.org" - -path_to_optimizer = "optimizers/spearmint_april2013_mod/" -version_info = ("# %76s #\n" % - "https://github.com/JasperSnoek/spearmint/tree/613350f2f617de3af5f101b1dc5eccf60867f67e") - - -def build_spearmint_call(config, options, optimizer_dir): - call = 'python ' + os.path.dirname(os.path.realpath(__file__)) + \ - "/" + path_to_optimizer + '/spearmint_sync.py' - call = ' '.join([call, optimizer_dir, - '--config', config.get('SPEARMINT', 'config'), - '--max-concurrent', config.get('DEFAULT', 'numberOfConcurrentJobs'), - '--max-finished-jobs', config.get('SPEARMINT', 'max_finished_jobs'), - '--polling-time', config.get('SPEARMINT', 'spearmint_polling_time'), - '--grid-size', config.get('SPEARMINT', 'grid_size'), - '--method', config.get('SPEARMINT', 'method'), - '--method-args=' + config.get('SPEARMINT', 'method_args'), - '--grid-seed', str(options.seed)]) - if config.get('SPEARMINT', 'method') != "GPEIChooser" and \ - config.get('SPEARMINT', 'method') != "GPEIOptChooser": - sys.stdout.write('WARNING: This chooser might not work yet\n') - call = ' '.join([call, config.get("SPEARMINT", 'method_args')]) - return call - - -#noinspection PyUnusedLocal -def restore(config, optimizer_dir, **kwargs): - """ - Returns the number of restored runs. This is the number of different configs - tested multiplied by the number of crossvalidation folds. - """ - restore_file = os.path.join(optimizer_dir, "expt-grid.pkl") - if not os.path.exists(restore_file): - print "Oups, this should have been checked before" - raise Exception("%s does not exist" % (restore_file,)) - sys.path.append(os.path.join( - os.path.dirname(os.path.realpath(__file__)), path_to_optimizer)) - import ExperimentGrid - # Assumes that all not valid states are marked as crashed - fh = open(restore_file) - exp_grid = cPickle.load(fh) - fh.close() - complete_runs = np.sum(exp_grid['status'] == 3) - restored_runs = complete_runs * config.getint('DEFAULT', 'numberCV') - try: - os.remove(os.path.join(optimizer_dir, "expt-grid.pkl.lock")) - except OSError: - pass - return restored_runs - - -#noinspection PyUnusedLocal -def main(config, options, experiment_dir, **kwargs): - # config: Loaded .cfg file - # options: Options containing seed, restore_dir, - # experiment_dir: Experiment directory/Benchmark_directory - # **kwargs: Nothing so far - - time_string = wrapping_util.get_time_string() - - # Find experiment directory - if options.restore: - if not os.path.exists(options.restore): - raise Exception("The restore directory does not exist") - optimizer_dir = options.restore - else: - optimizer_dir = os.path.join(experiment_dir, "spearmint_" \ - + str(options.seed) + "_" + time_string) - - # Build call - cmd = build_spearmint_call(config, options, optimizer_dir) - - # Set up experiment directory - if not os.path.exists(optimizer_dir): - os.mkdir(optimizer_dir) - # Make a link to the Protocol-Buffer config file - configpb = config.get('SPEARMINT', 'config') - if not os.path.exists(os.path.join(optimizer_dir, configpb)): - os.symlink(os.path.join(experiment_dir, "spearmint", configpb), - os.path.join(optimizer_dir, configpb)) - sys.stdout.write("### INFORMATION ################################################################\n") - sys.stdout.write("# You're running %40s #\n" % path_to_optimizer) - sys.stdout.write("%s" % version_info) - sys.stdout.write("# A newer version might be available, but not yet built in. #\n") - sys.stdout.write("# Please use this version only to reproduce our results on automl.org #\n") - sys.stdout.write("################################################################################\n") - return cmd, optimizer_dir \ No newline at end of file diff --git a/HPOlib/spearmint.py b/HPOlib/spearmint.py new file mode 120000 index 00000000..b96d4d3f --- /dev/null +++ b/HPOlib/spearmint.py @@ -0,0 +1 @@ +spearmint_april2013_mod.py \ No newline at end of file diff --git a/HPOlib/spearmint_april2013_mod.py b/HPOlib/spearmint_april2013_mod.py new file mode 100644 index 00000000..444c9e9b --- /dev/null +++ b/HPOlib/spearmint_april2013_mod.py @@ -0,0 +1,119 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#!/usr/bin/env python + +import cPickle +import os +import sys + +import numpy as np + +import wrapping_util + +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" + +path_to_optimizer = "optimizers/spearmint_april2013_mod/" +version_info = ("# %76s #\n" % + "https://github.com/JasperSnoek/spearmint/tree/613350f2f617de3af5f101b1dc5eccf60867f67e") + +optimizer_str = "spearmint_april2013_mod" + + +def build_spearmint_call(config, options, optimizer_dir): + call = 'python ' + os.path.dirname(os.path.realpath(__file__)) + \ + "/" + path_to_optimizer + '/spearmint_sync.py' + call = ' '.join([call, optimizer_dir, + '--config', config.get('SPEARMINT', 'config'), + '--max-concurrent', config.get('DEFAULT', 'numberOfConcurrentJobs'), + '--max-finished-jobs', config.get('SPEARMINT', 'max_finished_jobs'), + '--polling-time', config.get('SPEARMINT', 'spearmint_polling_time'), + '--grid-size', config.get('SPEARMINT', 'grid_size'), + '--method', config.get('SPEARMINT', 'method'), + '--method-args=' + config.get('SPEARMINT', 'method_args'), + '--grid-seed', str(options.seed)]) + if config.get('SPEARMINT', 'method') != "GPEIChooser" and \ + config.get('SPEARMINT', 'method') != "GPEIOptChooser": + sys.stdout.write('WARNING: This chooser might not work yet\n') + call = ' '.join([call, config.get("SPEARMINT", 'method_args')]) + return call + + +#noinspection PyUnusedLocal +def restore(config, optimizer_dir, **kwargs): + """ + Returns the number of restored runs. This is the number of different configs + tested multiplied by the number of crossvalidation folds. + """ + restore_file = os.path.join(optimizer_dir, "expt-grid.pkl") + if not os.path.exists(restore_file): + print "Oups, this should have been checked before" + raise Exception("%s does not exist" % (restore_file,)) + sys.path.append(os.path.join( + os.path.dirname(os.path.realpath(__file__)), path_to_optimizer)) + import ExperimentGrid + # Assumes that all not valid states are marked as crashed + fh = open(restore_file) + exp_grid = cPickle.load(fh) + fh.close() + complete_runs = np.sum(exp_grid['status'] == 3) + restored_runs = complete_runs * config.getint('DEFAULT', 'numberCV') + try: + os.remove(os.path.join(optimizer_dir, "expt-grid.pkl.lock")) + except OSError: + pass + return restored_runs + + +#noinspection PyUnusedLocal +def main(config, options, experiment_dir, **kwargs): + # config: Loaded .cfg file + # options: Options containing seed, restore_dir, + # experiment_dir: Experiment directory/Benchmark_directory + # **kwargs: Nothing so far + + time_string = wrapping_util.get_time_string() + + # Find experiment directory + if options.restore: + if not os.path.exists(options.restore): + raise Exception("The restore directory does not exist") + optimizer_dir = options.restore + else: + optimizer_dir = os.path.join(experiment_dir, optimizer_str + "_" + + str(options.seed) + "_" + time_string) + + # Build call + cmd = build_spearmint_call(config, options, optimizer_dir) + + # Set up experiment directory + if not os.path.exists(optimizer_dir): + os.mkdir(optimizer_dir) + # Make a link to the Protocol-Buffer config file + configpb = config.get('SPEARMINT', 'config') + if not os.path.exists(os.path.join(optimizer_dir, configpb)): + os.symlink(os.path.join(experiment_dir, optimizer_str, configpb), + os.path.join(optimizer_dir, configpb)) + sys.stdout.write("### INFORMATION ################################################################\n") + sys.stdout.write("# You're running %40s #\n" % path_to_optimizer) + sys.stdout.write("%s" % version_info) + sys.stdout.write("# A newer version might be available, but not yet built in. #\n") + sys.stdout.write("# Please use this version only to reproduce our results on automl.org #\n") + sys.stdout.write("################################################################################\n") + return cmd, optimizer_dir \ No newline at end of file From a197d954ac8a8f04f212433b6ddcfcab1e90e6a0 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 4 Feb 2014 13:48:24 +0100 Subject: [PATCH 012/139] Clean up plot.py update interface (has now the same like most of the other plotting scripts) and plots one parameter against performance --- HPOlib/Plotting/plot.py | 127 --------------------------- HPOlib/Plotting/plot_param.py | 156 ++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 127 deletions(-) delete mode 100644 HPOlib/Plotting/plot.py create mode 100644 HPOlib/Plotting/plot_param.py diff --git a/HPOlib/Plotting/plot.py b/HPOlib/Plotting/plot.py deleted file mode 100644 index 335095ce..00000000 --- a/HPOlib/Plotting/plot.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python - -import cPickle -import optparse -import os -import re -import sys - -from matplotlib.colors import LogNorm -from matplotlib.mlab import griddata -import matplotlib.pyplot as plt -import numpy as np - -__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] -__license__ = "3-clause BSD License" -__contact__ = "automl.org" - - -def translatePara(key, value): - # sanitize all params - newName = key - if "LOG10" in key: - pos = key.find("LOG10") - newName = key[0:pos] + key[pos+5:] - newName = newName.strip("_") - value = np.power(10, float(value)) - elif "LOG2" in key: - pos = key.find("LOG2") - newName = key[0:pos] + key[pos+4:] - newName = newName.strip("_") - value = np.power(2, float(value)) - elif "LOG" in key: - pos = key.find("LOG") - newName = key[0:pos] + key[pos+3:] - newName = newName.strip("_") - value = np.exp(float(value)) - #Check for Q value, returns round(x/q)*q - m = re.search('Q[0-999]{1,3}', key) - if m != None: - pos = newName.find(m.group(0)) - tmp = newName[0:pos] + newName[pos+3:] - newName = tmp.strip("_") - q = float(m.group(0)[1:]) - value = round(float(value)/q)*q - return (newName, value) - -def plot_params(paramDict, save="", title=""): - numPara = len(paramDict.keys()) - size = 10 - - if save == "": - # Show all paras in one plot - f = plt.figure(figsize = (1, 5), dpi = 100) - f.suptitle(title) - - plotNum = 1 - - for para in paramDict.keys(): - plt.subplot(numPara, 1, plotNum) - plotNum += 1 - plt.scatter(paramDict[para][:,0], paramDict[para][:,1], marker="^", \ - s=size, facecolors="none", edgecolors="k") - - plt.subplots_adjust(hspace=0.5) - plt.show() - else: - # Make one plot for each para - for para in paramDict.keys(): - fig, ax = plt.subplots(dpi=100) - fig.suptitle(title + " " + para) - ax.scatter(paramDict[para][:,0], paramDict[para][:,1], marker="^", \ - s=size, facecolors="none", edgecolors="k") - - save_fn = save + para - try: - fig.savefig(save_fn + ".png", dpi=100, facecolor='w', edgecolor='w', - orientation='portrait', papertype=None, format=None, - transparent=False, bbox_inches="tight", pad_inches=0.1) - fig.savefig(save_fn + ".pdf", dpi=100, facecolor='w', edgecolor='w', - orientation='portrait', papertype=None, format=None, - transparent=False, bbox_inches="tight", pad_inches=0.1) - fig.clear() - except Exception as e: - print e - -def main(): - parser = optparse.OptionParser() - parser.add_option("-s", "--save", dest = "save", default = "", \ - help = "Where to save plot instead of showing it?") - parser.add_option("-t", "--title", dest="title", default="TBA", \ - help = "Choose a supertitle for the plot") - (opts, args) = parser.parse_args() - sys.stdout.write("\nFound " + str(len(args)) + " arguments...\n") - paramDict = dict() - curOpt = args[0] - - if opts.title == "TBA": - title = curOpt - else: - title = opts.title - - for fn in args[1:]: - if not ".pkl" in fn: - print "This is not a .pkl file: %s" % (fn,) - continue - result_file = fn - fh = open(result_file, "r") - trials = cPickle.load(fh) - fh.close() - - keys = set() - for i in range(len(trials["trials"])): - for k in trials["trials"][i]["params"].keys(): - key, value = translatePara(k, trials["trials"][i]["params"][k]) - if key not in paramDict: - paramDict[key] = list() - else: - paramDict[key].append((value, trials["trials"][i]["result"])) - - for p in paramDict.keys(): - paramDict[p] = np.array(paramDict[p]) - - plot_params(paramDict, save=opts.save, title=title) - -if __name__ == "__main__": - main() - diff --git a/HPOlib/Plotting/plot_param.py b/HPOlib/Plotting/plot_param.py new file mode 100644 index 00000000..c6205322 --- /dev/null +++ b/HPOlib/Plotting/plot_param.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python + +import cPickle +import optparse +import re +import sys + +from matplotlib.pyplot import tight_layout, figure, subplots_adjust, subplot, savefig, show +from pylab import setp +from matplotlib.gridspec import GridSpec + +import numpy as np + + +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__license__ = "3-clause BSD License" +__contact__ = "automl.org" + + +def translate_para(key, value): + # sanitize all params + new_name = key + if "LOG10" in key: + pos = key.find("LOG10") + new_name = key[0:pos] + key[pos+5:] + new_name = new_name.strip("_") + value = np.power(10, float(value)) + elif "LOG2" in key: + pos = key.find("LOG2") + new_name = key[0:pos] + key[pos+4:] + new_name = new_name.strip("_") + value = np.power(2, float(value)) + elif "LOG" in key: + pos = key.find("LOG") + new_name = key[0:pos] + key[pos+3:] + new_name = new_name.strip("_") + value = np.exp(float(value)) + #Check for Q value, returns round(x/q)*q + m = re.search('Q[0-999]{1,3}', key) + if m is not None: + pos = new_name.find(m.group(0)) + tmp = new_name[0:pos] + new_name[pos+3:] + new_name = tmp.strip("_") + q = float(m.group(0)[1:]) + value = round(float(value)/q)*q + return new_name, value + + +def plot_params(value_list, result_list, name, save="", title="", jitter=0): + color = 'k' + marker = 'o' + size = 1 + + # Get handles + ratio = 5 + gs = GridSpec(ratio, 4) + fig = figure(1, dpi=100) + fig.suptitle(title, fontsize=16) + ax = subplot(gs[:, :]) + ax.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5) + + # Define xlims + min_y = min(result_list) + max_y = max(result_list) + y_offset = np.abs(max_y - min_y) * 0.1 + min_y -= y_offset + max_y += y_offset + + min_x = min(value_list) + max_x = max(value_list) + x_offset = np.abs(max_x - min_x) * 0.1 + min_x -= x_offset + max_x += x_offset + + # Maybe jitter data + # But before save best values + best_value = value_list[np.argmin(result_list)] + best_result = min(result_list) + + for idx in range(len(result_list)): + result_list[idx] += (np.random.rand(1) - 0.5) * jitter + value_list[idx] += (np.random.rand(1) - 0.5) * jitter + + # Plot + ax.scatter(value_list, result_list, facecolor=color, edgecolor=color, marker=marker, s=size*50) + ax.scatter(best_value, best_result, facecolor='r', + edgecolor='r', s=size*150, marker='o', alpha=0.5, + label="f[%5.3f]=%5.3f" % (best_value, best_result)) + + # Describe and label the stuff + ax.set_xlabel("Value of %s, jitter=%f" % (name, jitter)) + ax.set_ylabel("Minfunction value, jitter=%f" % jitter) + + leg = ax.legend(loc='best', fancybox=True, scatterpoints=1) + leg.get_frame().set_alpha(0.5) + + # Apply the xlims + ax.set_xlim([min_x, max_x]) + ax.set_ylim([min_y, max_y]) + + # Save Plot + tight_layout() + subplots_adjust(top=0.85) + if save != "": + savefig(save, dpi=100, facecolor='w', edgecolor='w', + orientation='portrait', papertype=None, format=None, + transparent=False, bbox_inches="tight", pad_inches=0.1) + else: + show() + + +def main(): + parser = optparse.OptionParser() + parser.add_option("-s", "--save", dest="save", default="", + help="Where to save plot instead of showing it?") + parser.add_option("-p", "--parameter", dest="param", default="", + help="Which parameter to plot") + parser.add_option("-j", "--jitter", dest="jitter", default=0, type=float, + help="Jitter data?") + parser.add_option("-t", "--title", dest="title", default="", + help="Choose a supertitle for the plot") + (opts, args) = parser.parse_args() + sys.stdout.write("\nFound " + str(len(args)) + " argument[s]...\n") + + title = opts.title + + value_list = list() + result_list = list() + param_set = set() + for fn in args: + if not ".pkl" in fn: + print "This is not a .pkl file: %s" % (fn,) + continue + print "Loading ", fn + fh = open(fn, "r") + trials = cPickle.load(fh) + fh.close() + + for t in trials["trials"]: + if opts.param in t["params"]: + k, value = translate_para(opts.param, t["params"][opts.param]) + value_list.append(float(value.strip("'"))) + result_list.append(t["result"]) + param_set.update(t["params"].keys()) + + if len(value_list) == 0: + print("No values found for param '%s', Available params:\n%s" % (opts.param, "\n".join([p for p in param_set]))) + sys.exit(1) + else: + print "Found %s values for %s" % (str(len(value_list)), opts.param) + + plot_params(value_list=value_list, result_list=result_list, name=opts.param, save=opts.save, title=title, + jitter=opts.jitter) + +if __name__ == "__main__": + main() From 79317f0a6acfb936f675d30bc0430a6c3805627f Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 4 Feb 2014 17:21:54 +0100 Subject: [PATCH 013/139] ENHANCEMENT: issue #18 and #4: setup.py adds a setup.py script which allows to install HPOlib. It is not yet possible to install runsolver and/or optimizers automatically as well. HPOlib-run is, having installed HPOlib, the script run an experiment. wrapping.py adds the HPOlib directory to the pythonpath to be able to run experiments without installing the lib. Other changes solve import problems. --- HPOlib-run | 26 ++++++ HPOlib/benchmarks/__init__.py | 2 + HPOlib/benchmarks/branin/__init__.py | 2 + HPOlib/benchmarks/har6/__init__.py | 2 + setup.py | 113 +++++++++++++++++++++++++++ 5 files changed, 145 insertions(+) create mode 100755 HPOlib-run create mode 100644 HPOlib/benchmarks/__init__.py create mode 100644 HPOlib/benchmarks/branin/__init__.py create mode 100644 HPOlib/benchmarks/har6/__init__.py create mode 100644 setup.py diff --git a/HPOlib-run b/HPOlib-run new file mode 100755 index 00000000..4e8ec3c3 --- /dev/null +++ b/HPOlib-run @@ -0,0 +1,26 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#!/usr/bin/env python + +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" + +from HPOlib import wrapping + +wrapping.main() \ No newline at end of file diff --git a/HPOlib/benchmarks/__init__.py b/HPOlib/benchmarks/__init__.py new file mode 100644 index 00000000..24cd9aaa --- /dev/null +++ b/HPOlib/benchmarks/__init__.py @@ -0,0 +1,2 @@ +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" \ No newline at end of file diff --git a/HPOlib/benchmarks/branin/__init__.py b/HPOlib/benchmarks/branin/__init__.py new file mode 100644 index 00000000..24cd9aaa --- /dev/null +++ b/HPOlib/benchmarks/branin/__init__.py @@ -0,0 +1,2 @@ +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" \ No newline at end of file diff --git a/HPOlib/benchmarks/har6/__init__.py b/HPOlib/benchmarks/har6/__init__.py new file mode 100644 index 00000000..24cd9aaa --- /dev/null +++ b/HPOlib/benchmarks/har6/__init__.py @@ -0,0 +1,2 @@ +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..c749a8ea --- /dev/null +++ b/setup.py @@ -0,0 +1,113 @@ +from __future__ import print_function +from setuptools import setup, find_packages +from setuptools.command.test import test as TestCommand +from setuptools.command.install import install as InstallCommand +import io +import codecs +import os +import sys + +import HPOlib + +here = os.path.abspath(os.path.dirname(__file__)) + +desc = 'A software that can be used to evaluate either black boxes or hyperparameter optimization algorithms' +long_desc = "HPOlib is a hyperparameter optimization library. It provides a common interface to three state of \n", \ + "the art hyperparameter optimization packages: SMAC, spearmint and hyperopt. Moreover, the library \n", \ + "provides optimization benchmarks which can be used to compare different hyperparameter optimization\n", \ + "packages and to establish standard test routines for hyperparameter optimization packages. " +keywords = 'hyperparameter optimization empirical evaluation black box' + + +def get_find_packages(): + packages = ['HPOlib', + 'HPOlib.config_parser', + 'HPOlib.benchmarks', + 'HPOlib.optimizers', + 'HPOlib.benchmarks.branin', + 'HPOlib.benchmarks.har6'] + #find_packages('.', exclude=('*.optimizers.*',)) + return packages + +package_dir = {'HPOlib': 'HPOlib', + 'HPOlib.config_parser': 'HPOlib/config_parser', + 'HPOlib.benchmarks': 'HPOlib/benchmarks', + 'HPOlib.optimizers': 'HPOlib/optimizers', + 'HPOlib.benchmarks.branin': 'HPOlib/benchmarks/branin', + 'HPOlib.benchmarks.har6': 'HPOlib/benchmarks/har6'} +package_data = {'HPOlib': ['*/params.txt', '*/space.py', '*/config.pb'], + 'HPOlib.benchmarks.branin': ['*.cfg', '*/params.txt', '*/space.py', '*/config.pb'], + 'HPOlib.benchmarks.har6': ['*.cfg', '*/params.txt', '*/space.py', '*/config.pb'], + 'HPOlib.config_parser': ['*.cfg']} +scripts = ['HPOlib-run'] + + +def read(*filenames, **kwargs): + encoding = kwargs.get('encoding', 'utf-8') + sep = kwargs.get('sep', '\n') + buf = [] + for filename in filenames: + with io.open(filename, encoding=encoding) as f: + buf.append(f.read()) + return sep.join(buf) + + +class InstallRunsolver(InstallCommand): + """Is supposed to install the runsolver later on""" + def run(self): + print("Hello User,\n" + "you need to install runsolver on your own :-)") + + +class PyTest(TestCommand): + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + print("Hello User,\n" + "so far we have not included any tests. Sorry") + +setup( + name='HPOlib', + version=HPOlib.__version__, + url='https://github.com/automl/HPOlib', + license='LGPLv3', + platforms=['Linux'], + author=HPOlib.__authors__, + tests_require=['unittest'], + install_requires=['argparse==1.2.1', + 'bson==0.3.3', + 'matplotlib==1.3.1', + 'networkx==1.8.1', + 'numpy==1.8.0', + 'protobuf==2.5.0', + 'scipy==0.13.2', + ], + cmdclass={'test': PyTest}, + author_email='eggenspk@informatik.uni-freiburg.de', + description=desc, + long_description=long_desc, + keywords=keywords, + packages=get_find_packages(), + package_dir=package_dir, + package_data=package_data, + #include_package_data=True, + scripts=scripts, + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Development Status :: 2 - Pre-Alpha', + 'Natural Language :: English', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)', + 'Operating System :: POSIX :: Linux', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Scientific/Engineering', + 'Topic :: Software Development', + ] +) From a53d41d387f80d3b2bbcdb0e2c168ea50643d75b Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Tue, 4 Feb 2014 17:33:02 +0100 Subject: [PATCH 014/139] REFACTOR: #5 Removed run_instance.py Removed the run_instance.py, respective code moved to runsolver_wrapper.py. --- HPOlib/benchmark_util.py | 8 +- HPOlib/benchmarks/branin/branin.py | 6 +- HPOlib/benchmarks/har6/har6.py | 8 +- HPOlib/cv.py | 4 +- HPOlib/run_instance.py | 242 ------------------- HPOlib/run_test.py | 105 --------- HPOlib/runsolver_wrapper.py | 359 ++++++++++++++++++----------- 7 files changed, 246 insertions(+), 486 deletions(-) delete mode 100644 HPOlib/run_instance.py delete mode 100644 HPOlib/run_test.py diff --git a/HPOlib/benchmark_util.py b/HPOlib/benchmark_util.py index 7b14a68d..2842ec9f 100644 --- a/HPOlib/benchmark_util.py +++ b/HPOlib/benchmark_util.py @@ -45,7 +45,8 @@ def parse_cli(): cli_args = sys.argv found_params = False skip = True - for idx, arg in enumerate(cli_args): + iterator = enumerate(cli_args) + for idx, arg in iterator: if skip: skip = False continue @@ -67,15 +68,12 @@ def parse_cli(): "--params argument. Please change the order.") elif arg[0] == "-" and arg[0:2] != "--" and found_params: - if cli_args[idx+1][0] == "-": - raise ValueError("Hyperparameter name is not allowed to have a " - "leading minus %s" % cli_args[idx + 1]) parameters[cli_args[idx][1:]] = cli_args[idx+1] elif arg[0] == "-" and arg[0:2] != "--" and not found_params: raise ValueError("You either try to use arguments with only one lea" "ding minus or try to specify a hyperparameter bef" - "ore the --params argument.") + "ore the --params argument. %s" % " ".join(cli_args)) elif not found_params: raise ValueError("Illegal command line string, expected an argument" diff --git a/HPOlib/benchmarks/branin/branin.py b/HPOlib/benchmarks/branin/branin.py index 46deb67c..8d276526 100644 --- a/HPOlib/benchmarks/branin/branin.py +++ b/HPOlib/benchmarks/branin/branin.py @@ -19,6 +19,7 @@ import sys from numpy import NaN, pi, cos, ndarray import benchmark_util +import time __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" @@ -56,6 +57,9 @@ def main(params, **kwargs): return y if __name__ == "__main__": + starttime = time.time() args, params = benchmark_util.parse_cli() result = main(params, **args) - print "Result", result \ No newline at end of file + duration = time.time() - starttime + print "Result for ParamILS: %s, %d, 1, %f, %d, %s" % \ + ("SAT", abs(duration), result, -1, str(__file__)) diff --git a/HPOlib/benchmarks/har6/har6.py b/HPOlib/benchmarks/har6/har6.py index ac0aa0c6..82441b55 100644 --- a/HPOlib/benchmarks/har6/har6.py +++ b/HPOlib/benchmarks/har6/har6.py @@ -18,6 +18,8 @@ from numpy import NaN, array, ndarray, exp import benchmark_util +import sys +import time __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" @@ -85,6 +87,10 @@ def main(params, **kwargs): if __name__ == "__main__": + print sys.argv + starttime = time.time() args, params = benchmark_util.parse_cli() result = main(params, **args) - print "Result", result \ No newline at end of file + duration = time.time() - starttime + print "Result for ParamILS: %s, %d, 1, %f, %d, %s" % \ + ("SAT", abs(duration), result, -1, str(__file__)) \ No newline at end of file diff --git a/HPOlib/cv.py b/HPOlib/cv.py index 69e86ce3..b0b97a0c 100644 --- a/HPOlib/cv.py +++ b/HPOlib/cv.py @@ -238,8 +238,8 @@ def read_params_from_command_line(): key = key[1:] if value[0] != "'" or value[-1] != "'": - raise ValueError("Expected parameter value %d to be inside single" - "quotation marks." % (idx*2 + 1)) + raise ValueError("Expected parameter value %s to be inside single" + "quotation marks." % value) else: value = value.strip("'") params[key] = value diff --git a/HPOlib/run_instance.py b/HPOlib/run_instance.py deleted file mode 100644 index 6238d7d2..00000000 --- a/HPOlib/run_instance.py +++ /dev/null @@ -1,242 +0,0 @@ -## -# wrapping: A program making it easy to use hyperparameter -# optimization software. -# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -#!/usr/bin/env python - -import cPickle -import imp -from optparse import OptionParser -import os -import re -import sys -import time - -import numpy as np - -from config_parser.parse import parse_config -import Experiment -import wrapping_util - -__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] -__contact__ = "automl.org" - - -def loadExperimentFile(pickle): - experiment = Experiment.Experiment(os.path.split(pickle)[0], - os.path.split(pickle)[1].split(".")[0]) - return experiment - - -def remove_param_metadata(params): - """ - Check whether some params are defined on the Log scale or with a Q value, - must be marked with "LOG$_{paramname}" or Q[0-999]_$paramname - LOG/Q will be removed from the paramname - """ - for para in params: - new_name = para - if "LOG10_" in para: - pos = para.find("LOG10") - new_name = para[0:pos] + para[pos+6:] - # new_name = new_name.strip("_") - params[new_name] = np.power(10, float(params[para])) - del params[para] - elif "LOG2" in para: - pos = para.find("LOG2_") - new_name = para[0:pos] + para[pos+5:] - # new_name = new_name.strip("_") - params[new_name] = np.power(2, float(params[para])) - del params[para] - elif "LOG_" in para: - pos = para.find("LOG") - new_name = para[0:pos] + para[pos+4:] - # new_name = new_name.strip("_") - params[new_name] = np.exp(float(params[para])) - del params[para] - #Check for Q value, returns round(x/q)*q - m = re.search(r'Q[0-999\.]{1,10}_', para) - if m is not None: - pos = new_name.find(m.group(0)) - tmp = new_name[0:pos] + new_name[pos+len(m.group(0)):] - #tmp = tmp.strip("_") - q = float(m.group(0)[1:-1]) - params[tmp] = round(float(params[new_name])/q)*q - del params[new_name] - - -def run_instance(fold, fn, params, cfg): - # Run instance - result = np.NaN - status = "UNSAT" - folds = cfg.getint('DEFAULT', 'numberCV') - - # Remove additional information about variables and change them accordingly - remove_param_metadata(params) - - starttime = time.time() - try: - result = fn(params, fold=fold, folds=folds) - status = "SAT" - except Exception: - print wrapping_util.format_traceback(sys.exc_info()) - status = "CRASHED" - duration = time.time() - starttime - return result, duration, status - - -def get_trial_index(experiment, fold, params): - # Check whether we are in a new configuration; This has to check whether - # the params were already inserted but also whether the fold already run - # This is checked twice; the instance_result has to be not NaN and the - # entry in instance_order has to exist - new = True - trial_index = np.NaN - for idx, trial in enumerate(experiment.trials): - exp = trial['params'] - if exp == params and (idx, fold) not in experiment.instance_order and \ - (experiment.get_trial_from_id(idx)['instance_results'][fold] == np.NaN or \ - experiment.get_trial_from_id(idx)['instance_results'][fold] != - experiment.get_trial_from_id(idx)['instance_results'][fold]): - new = False - trial_index = idx - break - if new: - trial_index = experiment.add_job(params) - return trial_index - - -def main(): - # Parse options and arguments - usage = "This script loads the module specified in the config.cfg" + \ - "file as 'function'.\nIt will either run the main() method or the"+\ - " run_test() method and print the output as: \n" + \ - "'Result for ParamILS: , , , " + \ - ", , ' \n\n" + \ - "%prog [-f -c | -d] \n" + \ - "where is a pickled dict of parameters\n" - parser = OptionParser(usage=usage) - parser.add_option("-f", "--fold", - action="store", dest="fold", type="int", default=None, - help="Number of fold to execute") - parser.add_option("-c", "--config", - action="store", dest="configFile", default=None, - help="path to the config.cfg") - parser.add_option("-s", "--seed", - action="store", dest="seed", type="int", default=1, - help="Seed to use for calculations") - parser.add_option("-p", "--pkl", - action="store", dest="pkl", default=None, - help="Pickle file for bookkeeping") - parser.add_option("-t", "--test", - action = "store_true", dest = "test", default = False, - help = "With this flag activated a function run_test " + \ - "is called instead of main") - (options, args) = parser.parse_args() - - if (options.configFile is None and options.fold is not None) or \ - (options.configFile is not None and options.fold is None): - parser.error("If not using -d, fold and config.cfg need " + \ - "to be specified") - if options.configFile is None and options.fold is None and \ - not options.debug: - parser.print_usage() - - seed = options.seed - - # Load the pickled param file... - if len(args) != 1: - raise Exception("Too many or less paramfiles specified") - paramfile = args[0] - print "Paramfile", paramfile, os.path.exists(paramfile) - if not os.path.exists(paramfile): - raise Exception("%s is not a file\n" % paramfile) - fh = open(paramfile, "r") - params = cPickle.load(fh) - fh.close() - - # ...then load config.cfg... - cfg = parse_config(options.configFile, allow_no_value=True) - if os.path.isabs(cfg.get("DEFAULT", "function")): - fn_path = cfg.get("DEFAULT", "function") - else: - fn_path = cfg.get("DEFAULT", "function") - fn_path_parent = os.path.join("..", cfg.get("DEFAULT", "function")) - fn_name, ext = os.path.splitext(os.path.basename(fn_path)) - try: - fn = imp.load_source(fn_name, fn_path) - except (ImportError, IOError) as e: - print "Raised", e, "trying to recover..." - try: - print os.path.exists(fn_path_parent) - fn = imp.load_source(fn_name, fn_path_parent) - except (ImportError, IOError): - print os.path.join(fn_path_parent) - print(("Could not find\n%s\n\tin\n%s\n\tor its parent directory " + - "relative to\n%s") - % (fn_name, fn_path, os.getcwd())) - import traceback - print traceback.format_exc() - sys.exit(1) - - if options.test: - fn = fn.run_test - else: - fn = fn.main - - # Do bookkeeping before running the instance - optimizer = "w/o bookkeeping" - - if options.pkl is not None: - # Don't do bookkeeping if you don't want to - if not os.path.isfile(options.pkl): - raise Exception("%s does not exist\n" % options.pkl) - experiment = loadExperimentFile(options.pkl) - optimizer = experiment.optimizer - - # This has the side-effect of adding a job - trial_index = get_trial_index(experiment, options.fold, params) - - experiment.set_one_fold_running(trial_index, options.fold) - del experiment # release lock - - # TODO: Forward seed to run_instance, because like this it is useless - result, duration, status = run_instance(options.fold, fn, params, cfg) - - # Do bookkeeping after the run_instance - if options.pkl is not None: - # Don't do bookkeeping if we don't want you to do - if not os.path.isfile(options.pkl): - raise Exception("%s does not exist\n" % paramfile) - experiment = loadExperimentFile(options.pkl) - optimizer = experiment.optimizer - if status == "SAT": - experiment.set_one_fold_complete(trial_index, options.fold, result, - duration) - elif status == "CRASHED": - result = cfg.getfloat("DEFAULT", "result_on_terminate") - experiment.set_one_fold_crashed(trial_index, options.fold, result, duration) - else: - # TODO: We need a global stopping mechanism - sys.exit(1) - del experiment #release lock - - print "Result for ParamILS: %s, %d, 1, %f, %d, %s for %s" % \ - (status, abs(duration), result, seed, optimizer, str(fn_path)) - -if __name__ == "__main__": - main() diff --git a/HPOlib/run_test.py b/HPOlib/run_test.py deleted file mode 100644 index d237f8b3..00000000 --- a/HPOlib/run_test.py +++ /dev/null @@ -1,105 +0,0 @@ -## -# wrapping: A program making it easy to use hyperparameter -# optimization software. -# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -#!/usr/bin/env python - -import cPickle -from importlib import import_module -from optparse import OptionParser -import os -import re -import subprocess -import sys -import time - -from gzip import GzipFile as gfile -import numpy as np - -from config_parser.parse import parse_config -from Experiment import Experiment -import wrapping_util - -__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] -__contact__ = "automl.org" - - -def loadExperimentFile(): - optimizer = os.getcwd().split("/")[-1].split("_")[0] - experiment = Experiment(".", optimizer) - return experiment - -def main(): - # Parse options and arguments - usage = "Coming soon" - parser = OptionParser(usage=usage) - (options, args) = parser.parse_args() - - if len(args) != 1: - raise Exception("You must specify a directory.") - - # Then load config.cfg and function - cfg = parse_config("config.cfg", allow_no_value=True) - module = cfg.get("DEFAULT", "function") - - os.chdir(args[0]) - - # Load the experiment pickle and extract best configuration according to the - # validation loss - experiment = loadExperimentFile() - optimizer = experiment.optimizer - if optimizer == "smac": - tmp_results = experiment.instance_results - tmp_results = np.ma.masked_array(tmp_results, np.isnan(tmp_results)) - results = np.mean(tmp_results, axis=1) - else: - tmp_results = experiment.results - tmp_results[np.isnan(tmp_results)] = cfg.get('DEFAULT', 'result_on_terminate') - results = tmp_results - best_config_idx = np.argmin(results) - print "Found best config #%d with validation loss %f" % (best_config_idx, results[best_config_idx]) - params = experiment.params[best_config_idx] - del experiment #release lock - - # Create a param pickle - time_string = wrapping_util.get_time_string() - params_filename = os.path.join(os.getcwd(), "params" + time_string) - params_fh = open(params_filename, 'w') - print "Pickling param dict", params_filename, params - cPickle.dump(params, params_fh) - params_fh.close() - - #Call run_instance.py - fh = open(args[0][0:-1] + "_test_run.out", "w") - leading_runsolver_info = cfg.get('DEFAULT', 'leading_runsolver_info') - leading_algo_info = cfg.get('DEFAULT', 'leading_algo_info') - leading_algo_info = "optirun" - cmd = "%s %s python %s --fold 1 --config ../config.cfg --test %s" % (leading_runsolver_info, leading_algo_info, os.path.join(os.path.dirname(os.path.realpath(__file__)) + "/run_instance.py"), params_filename) - process = subprocess.Popen(cmd, stdout=fh, stderr=fh, - shell=True, executable="/bin/bash") - - print - print cmd - print "-----------------------RUNNING TEST----------------------------" - ret = process.wait() - fh.close() - - os.remove(params_filename) - print ret - -if __name__ == "__main__": - main() - #test() diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 1bb70e94..7108cdd7 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -16,14 +16,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python - -import cPickle +from collections import OrderedDict +import imp import numpy as np import os +import re +import subprocess import sys import time -import subprocess import Experiment import wrapping_util @@ -38,31 +38,49 @@ def get_optimizer(): return "_".join(os.getcwd().split("/")[-1].split("_")[0:-2]) -def load_experiment_file(seed): +def remove_param_metadata(params): + """ + Check whether some params are defined on the Log scale or with a Q value, + must be marked with "LOG$_{paramname}" or Q[0-999]_$paramname + LOG/Q will be removed from the paramname + """ + for para in params: + new_name = para + if "LOG10_" in para: + pos = para.find("LOG10") + new_name = para[0:pos] + para[pos+6:] + # new_name = new_name.strip("_") + params[new_name] = np.power(10, float(params[para])) + del params[para] + elif "LOG2" in para: + pos = para.find("LOG2_") + new_name = para[0:pos] + para[pos+5:] + # new_name = new_name.strip("_") + params[new_name] = np.power(2, float(params[para])) + del params[para] + elif "LOG_" in para: + pos = para.find("LOG") + new_name = para[0:pos] + para[pos+4:] + # new_name = new_name.strip("_") + params[new_name] = np.exp(float(params[para])) + del params[para] + #Check for Q value, returns round(x/q)*q + m = re.search(r'Q[0-999\.]{1,10}_', para) + if m is not None: + pos = new_name.find(m.group(0)) + tmp = new_name[0:pos] + new_name[pos+len(m.group(0)):] + #tmp = tmp.strip("_") + q = float(m.group(0)[1:-1]) + params[tmp] = round(float(params[new_name])/q)*q + del params[new_name] + + +def load_experiment_file(): optimizer = get_optimizer() experiment = Experiment.Experiment(".", optimizer) return experiment -def replace_nan_in_last_trial(replacement): - exp = load_experiment_file() - # TODO: Make a better test to find out if we are supposed to write to exp. - if len(exp.trials) == 0 and len(exp.instance_order) == 0: - del exp - else: - _id, fold = exp.instance_order[-1] - if not np.isfinite(exp.trials[_id]['instance_results'][fold]): - exp.trials[_id]['instance_results'][fold] = replacement - exp.trials[_id]['instance_status'][fold] == Experiment.BROKEN_STATE - elif exp.trials[_id]['instance_results'][fold] == replacement: - pass - else: - raise Exception("Trying to replace %f with %f" % - (exp.trials[_id]['instance_results'][fold], replacement)) - exp._save_jobs() - del exp # This also saves the experiment - - def read_runsolver_output(runsolver_output_file): """ Read the runsolver output, watch out for @@ -73,10 +91,12 @@ def read_runsolver_output(runsolver_output_file): In case one of these happened, send back the worst possible result as specified in the config """ + limit_exceeded = None + cpu_time = None + wallclock_time = None + solver_ended_section = False with open(runsolver_output_file, 'r') as f: runsolver_output_content = f.readlines() - limit_exceeded = None - error_time = 0 for line in runsolver_output_content: if "Maximum CPU time exceeded" in line: limit_exceeded = "CPU time exceeded" @@ -86,10 +106,13 @@ def read_runsolver_output(runsolver_output_file): limit_exceeded = "VSize exceeded" if "Mem limit exceeded" in line: limit_exceeded = "Memory exceeded" - if "Real time (s): " in line: - # If the solver terminated, get the wallclock time of termination - error_time = float(line.split()[3]) - return limit_exceeded, error_time + if "Solver just ended. Dumping a history of the last" in line: + solver_ended_section = True + if "Real time (s): " in line and solver_ended_section: + wallclock_time = float(line.split()[3]) + if "CPU time (s): " in line and solver_ended_section: + cpu_time = float(line.split()[3]) + return cpu_time, wallclock_time, limit_exceeded def read_run_instance_output(run_instance_output): @@ -103,18 +126,43 @@ def read_run_instance_output(run_instance_output): fh.close() result_string = None for line in run_instance_content: - pos = line.find("Result for ParamILS:") - if pos != -1: - result_string = line[pos:] + match = re.search(r"\s*[Rr]esult\s+(?:([Ff]or)|([oO]f))\s" + r"+(?:(HAL)|(ParamILS)|(SMAC)|([tT]his [wW]rapper))", + line) + if match: + pos = match.start(0) + result_string = line[pos:].strip() result_array = result_string.split() + result_array = [value.strip(",") for value in result_array] break return result_array, result_string -def main(): +def get_trial_index(experiment, fold, params): + # Check whether we are in a new configuration; This has to check whether + # the params were already inserted but also whether the fold already run + # This is checked twice; the instance_result has to be not NaN and the + # entry in instance_order has to exist + new = True + trial_index = np.NaN + for idx, trial in enumerate(experiment.trials): + exp = trial['params'] + if exp == params and (idx, fold) not in experiment.instance_order and \ + (experiment.get_trial_from_id(idx)['instance_results'][fold] == np.NaN or \ + experiment.get_trial_from_id(idx)['instance_results'][fold] != + experiment.get_trial_from_id(idx)['instance_results'][fold]): + new = False + trial_index = idx + break + if new: + trial_index = experiment.add_job(params) + return trial_index + + +def parse_command_line(): # Parse options and arguments - usage = "This script pickles the params and runs the runsolver with " +\ + usage = "This script pickles the params and runs the runsolver with " + \ "run_instance and extract the output for the optimizer \n" + \ "The output is printed im a SMACish way: \n\n" + \ "'Result for ParamILS: , , , " + \ @@ -131,61 +179,147 @@ def main(): # Then get some information for run_instance fold = int(sys.argv[1]) seed = int(sys.argv[5]) + return fold, seed - optimizer = get_optimizer() - - # This has to be done here for SMAC, since smac does not call cv.py - if 'smac' in optimizer: - cv_starttime = time.time() - experiment = load_experiment_file(seed) - experiment.start_cv(cv_starttime) - del experiment - - # Again we need to find the config.cfg - cfg = cfg = wrapping_util.load_experiment_config_file() - cfg_filename = "config.cfg" - # Ignore smac cutofftime +def get_function_filename(cfg): + if os.path.isabs(cfg.get("DEFAULT", "function")): + fn_path = cfg.get("DEFAULT", "function") + else: + fn_path = cfg.get("DEFAULT", "function") + fn_path_parent = os.path.join("..", cfg.get("DEFAULT", "function")) + fn_name, ext = os.path.splitext(os.path.basename(fn_path)) + try: + fn = imp.load_source(fn_name, fn_path) + except (ImportError, IOError) as e: + print "Raised", e, "trying to recover..." + try: + print os.path.exists(fn_path_parent) + fn = imp.load_source(fn_name, fn_path_parent) + except (ImportError, IOError): + print os.path.join(fn_path_parent) + print(("Could not find\n%s\n\tin\n%s\n\tor its parent directory " + + "relative to\n%s") + % (fn_name, fn_path, os.getcwd())) + import traceback + + print traceback.format_exc() + sys.exit(1) + fn_filename = os.path.realpath(fn.__file__) + if fn_filename[-3:] == "pyc": + fn_filename = fn_filename[:-1] + + return fn_filename + + +def make_command(cfg, fold, param_string, run_instance_output): time_limit = cfg.getint('DEFAULT', 'runsolver_time_limit') memory_limit = cfg.getint('DEFAULT', 'memory_limit') cpu_limit = cfg.getint('DEFAULT', 'cpu_limit') + fn_filename = get_function_filename(cfg) + python_cmd = cfg.get("DEFAULT", "leading_algo_info") + " python " + \ + fn_filename + " --fold %d --folds %d --params %s" % \ + (fold, cfg.getint("DEFAULT", "numberCV"), param_string) + # Do not write the actual task in quotes because runsolver will not work + # then; also we need use-pty antd timestamp so that the "solver" output + # is flushed to the output directory + delay = 0 + cmd = cfg.get("DEFAULT", "leading_runsolver_info") + \ + " runsolver -o %s --timestamp --use-pty -W %d -C %d -M %d -d %d %s" \ + % (run_instance_output, time_limit, cpu_limit, memory_limit, delay, + python_cmd) + return cmd - # Now build param dict - param_list = sys.argv[6:] - params = dict() - for idx, i in enumerate(param_list[0::2]): - params[param_list[idx*2][1:]] = (param_list[idx*2+1].strip("'")) - # TODO: remove overhead of pickling the param dict - time_string = wrapping_util.get_time_string() - params_filename = os.path.join(os.getcwd(), "params" + time_string) - params_fh = open(params_filename, 'w') - print 'Pickling param dict', params_filename, params - cPickle.dump(params, params_fh) - params_fh.close() +def get_parameters(): + params = dict(zip(sys.argv[6::2], sys.argv[7::2])) + remove_param_metadata(params) + params = OrderedDict(sorted(params.items(), key=lambda t: t[0])) + return params - # Timestamp and use-pty are options are used so that output of the "solver" - # is flushed to the solver output file directly - run_instance_output = os.path.join(os.getcwd(), time_string + "_run_instance.out") - sys.stdout.write("Using optimizer: " + str(optimizer) + "\n") - python_cmd = cfg.get("DEFAULT", "leading_algo_info") + " python " + \ - os.path.join(os.path.dirname(os.path.realpath(__file__)), - "run_instance.py") + \ - " --fold %d --config %s --seed %d --pkl %s" % \ - (fold, cfg_filename, seed, optimizer + ".pkl") +def parse_output_files(cfg, run_instance_output, runsolver_output_file): + cpu_time, wallclock_time, error = read_runsolver_output( + runsolver_output_file) + result_array, result_string = read_run_instance_output(run_instance_output) - # Do not write the actual task in quotes because runsolver will not work - # then - delay = 0 - cmd = cfg.get("DEFAULT", "leading_runsolver_info") + \ - " runsolver -o %s --timestamp --use-pty -W %d -C %d -M %d -d %d %s %s" \ - % (run_instance_output, time_limit, cpu_limit, memory_limit, delay, - python_cmd, params_filename) + if error is None and result_string is None: + additional_data = "No result string returned. Please have a look " \ + "at " + run_instance_output + rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("DEFAULT", + "result_on_terminate"), additional_data) + os.remove(runsolver_output_file) + + elif error is None and result_array[3] != "SAT": + additional_data = "Please have a look at " + run_instance_output + "."\ + "The output status is not \"SAT\"" + rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("DEFAULT", + "result_on_terminate"), additional_data) + os.remove(runsolver_output_file) + + elif error is None and not np.isfinite(float(result_array[6].strip(","))): + additional_data = "Response value is not finite. Please have a look " \ + "at " + run_instance_output + rval = (cpu_time, wallclock_time, "UNSAT", cfg.getfloat("DEFAULT", + "result_on_terminate"), additional_data) + elif error is None: + # Remove the run_instance_output only if there is a valid result + os.remove(run_instance_output) + os.remove(runsolver_output_file) + rval = (cpu_time, wallclock_time, "SAT", float(result_array[6].strip(",")), + cfg.get("DEFAULT", "function")) + print result_string + + else: + rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("DEFAULT", + "result_on_terminate"), error + " Please have a look at " + + runsolver_output_file) + # It is useful to have the run_instance_output for debugging + os.remove(run_instance_output) + + return rval + + +def format_return_string(status, runtime, runlength, quality, seed, + additional_data): + return_string = "Result for ParamILS: %s, %f, %d, %f, %d, %s" %\ + (status, runtime, runlength, quality, seed, additional_data) + return return_string + + +def main(): + optimizer = get_optimizer() + # This has to be done here for SMAC, since smac does not call cv.py + if 'smac' in optimizer: + cv_starttime = time.time() + experiment = load_experiment_file() + experiment.start_cv(cv_starttime) + del experiment + + cfg = wrapping_util.load_experiment_config_file() + + fold, seed = parse_command_line() + # Side-effect: removes all additional information like log and applies + # transformations to the parameters + params = get_parameters() + param_string = " ".join([key + " " + params[key] for key in params]) + + time_string = wrapping_util.get_time_string() + run_instance_output = os.path.join(os.getcwd(), + time_string + "_run_instance.out") runsolver_output_file = os.path.join(os.getcwd(), time_string + "_runsolver.out") + + cmd = make_command(cfg, fold, param_string, run_instance_output) + fh = open(runsolver_output_file, "w") + experiment = load_experiment_file() + # Side-effect: adds a job if it is not yet in the experiments file + trial_index = get_trial_index(experiment, fold, params) + experiment.set_one_fold_running(trial_index, fold) + del experiment # release Experiment lock + process = subprocess.Popen(cmd, stdout=fh, stderr=fh, shell=True, executable="/bin/bash") @@ -195,66 +329,31 @@ def main(): process.wait() fh.close() - limit_exceeded, error_time = read_runsolver_output(runsolver_output_file) - # , , , , , - error_string = "Result for ParamILS: %s, %d, 0, %f, %d, %s" - - result_array, result_string = read_run_instance_output(run_instance_output) - - # Write the SMACish output to the command line and remove temporary files - # Additionaly, we also have to replace the NaN-value in the result file by - # the worst possible value - if result_array is not None and result_array[3].strip(",") not in ("SAT", "CRASHED"): - raise Exception("Unknown return status %s" % result_array[3].strip(",")) - - if (limit_exceeded is None and result_string is None) or\ - (result_string is not None and "CRASHED" in result_string and not limit_exceeded): - replace_nan_in_last_trial(cfg.getfloat("DEFAULT", "result_on_terminate")) - return_string = error_string % ("SAT", error_time, - cfg.getfloat('DEFAULT', "result_on_terminate"), - seed, - "Please have a look at " + run_instance_output) - print return_string - os.remove(runsolver_output_file) - - elif limit_exceeded is None: - result_float = float(result_array[6].strip(",")) - if not np.isfinite(result_float): - replace_nan_in_last_trial() - experiment = load_experiment_file(seed) - last_experiment = experiment.instance_order[-1] - assert last_experiment is not None - experiment.instance_results[last_experiment[0]][last_experiment[1]]\ - = cfg.getfloat('DEFAULT', "result_on_terminate") - del experiment # This also saves the experiment - - result_array[6] = (cfg.getfloat("DEFAULT", "result_on_terminate")) + "," - result_string = " ".join(result_array) - else: - # Remove the run_instance_output only if there is a valid result - os.remove(run_instance_output) - os.remove(runsolver_output_file) - return_string = result_string - print result_string - + cpu_time, wallclock_time, status, result, additional_data = \ + parse_output_files(cfg, run_instance_output, runsolver_output_file) + + experiment = load_experiment_file() + if status == "SAT": + experiment.set_one_fold_complete(trial_index, fold, result, + wallclock_time) + elif status == "CRASHED" or status == "UNSAT": + result = cfg.getfloat("DEFAULT", "result_on_terminate") + experiment.set_one_fold_crashed(trial_index, fold, result, + wallclock_time) else: - return_string = error_string % ("SAT", error_time, cfg.getfloat - ('DEFAULT', "result_on_terminate"), seed, - limit_exceeded) - print return_string - # It is useful to have the run_instance_output for debugging - #os.remove(run_instance_output) + # TODO: We need a global stopping mechanism + pass + del experiment #release lock - os.remove(params_filename) + return_string = format_return_string(status, wallclock_time, 1, result, + seed, additional_data) - # Remove param pkl and runsolver files - #os.remove(run_instance_output) - #os.remove(runsolver_output_file) - #os.remove(params_filename) if 'smac' in optimizer: - experiment = load_experiment_file(seed) + experiment = load_experiment_file() experiment.end_cv(time.time()) del experiment + + print return_string return return_string if __name__ == "__main__": From 3f23f3c29c0cfcf8e7c18cbd6719a823b27da311 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 5 Feb 2014 17:26:36 +0100 Subject: [PATCH 015/139] FIX: Fixed the branin result for a crash The result for a crash was unreasonable low (1). This could cause the optimization algorithm to work in the wrong direction. --- HPOlib/benchmarks/branin/config.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HPOlib/benchmarks/branin/config.cfg b/HPOlib/benchmarks/branin/config.cfg index 85f55f8f..3b99804a 100644 --- a/HPOlib/benchmarks/branin/config.cfg +++ b/HPOlib/benchmarks/branin/config.cfg @@ -4,4 +4,4 @@ p = smac/params.txt [DEFAULT] function = branin.py numberOfJobs = 200 -result_on_terminate = 1 +result_on_terminate = 1000 From 892d3ba96f58c05bb8623ee7194836ae6617fa34 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 5 Feb 2014 18:46:01 +0100 Subject: [PATCH 016/139] FIX: SMAC parameter intensification percentage=0 This makes SMAC (hopefully) deterministic If the intensification percentage is larger than 0, then an amount of time greater than zero seconds is spent on intensification. The amount of time depends on the time needed to build the model and is therefore not deterministic. It can also occur that the number of target algorithm runs in the intensification timespan differs across runs. We performed tests on branin and har6 and they resulted in a small performance gain compared to the values on the NIPS workshop poster. --- HPOlib/config_parser/smac_2_06_01-devDefault.cfg | 1 + HPOlib/config_parser/smac_2_06_01-dev_parser.py | 3 ++- HPOlib/smac_2_06_01-dev.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HPOlib/config_parser/smac_2_06_01-devDefault.cfg b/HPOlib/config_parser/smac_2_06_01-devDefault.cfg index 5eaa6630..4ba62d6c 100644 --- a/HPOlib/config_parser/smac_2_06_01-devDefault.cfg +++ b/HPOlib/config_parser/smac_2_06_01-devDefault.cfg @@ -25,5 +25,6 @@ runtimeLimit = 2147483647 deterministic = True retryTargetAlgorithmRunCount = 0 +intensification_percentage = 0 version = smac_2_06_01-dev \ No newline at end of file diff --git a/HPOlib/config_parser/smac_2_06_01-dev_parser.py b/HPOlib/config_parser/smac_2_06_01-dev_parser.py index 7caaf6d7..4b108e64 100644 --- a/HPOlib/config_parser/smac_2_06_01-dev_parser.py +++ b/HPOlib/config_parser/smac_2_06_01-dev_parser.py @@ -45,7 +45,8 @@ def add_default(config): 'testInstanceFile', 'p', 'rf_full_tree_bootstrap', 'rf_split_min', 'adaptiveCapping', 'maxIncumbentRuns', 'numIterations', 'runtimeLimit', 'deterministic', - 'retryTargetAlgorithmRunCount'): + 'retryTargetAlgorithmRunCount', + 'intensification_percentage'): if not config.has_option('SMAC', option): config.set('SMAC', option, smac_config.get('SMAC', option)) diff --git a/HPOlib/smac_2_06_01-dev.py b/HPOlib/smac_2_06_01-dev.py index 0ad317e5..2ae032a2 100644 --- a/HPOlib/smac_2_06_01-dev.py +++ b/HPOlib/smac_2_06_01-dev.py @@ -88,6 +88,8 @@ def build_smac_call(config, options, optimizer_dir): '--retryTargetAlgorithmRunCount', config.get('SMAC', 'retryTargetAlgorithmRunCount'), '--save-runs-every-iteration true', + '--intensification-percentage', + config.get('SMAC', 'intensification_percentage'), '--rf-split-min', config.get('SMAC', 'rf_split_min')]) if config.getboolean('SMAC', 'deterministic'): From ad2c195c4abf053bb323ee90cde1bdba96970d7b Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Thu, 6 Feb 2014 15:36:06 +0100 Subject: [PATCH 017/139] FIX: imports for issue #18 and #4 --- HPOlib/Experiment.py | 3 +- HPOlib/Locker.py | 1 + HPOlib/Plotting/plot_param.py | 1 - HPOlib/__init__.py | 4 +- HPOlib/benchmarks/branin/branin.py | 75 ++++++++++++++++++++++++++++-- HPOlib/benchmarks/har6/har6.py | 3 +- HPOlib/check_before_start.py | 2 - HPOlib/config_parser/parse.py | 2 - HPOlib/cv.py | 11 ++--- HPOlib/hyperopt_august2013_mod.py | 15 +++--- HPOlib/randomtpe.py | 3 +- HPOlib/run_instance.py | 5 +- HPOlib/run_test.py | 10 ++-- HPOlib/runsolver_wrapper.py | 4 +- HPOlib/smac_2_06_01-dev.py | 5 +- HPOlib/spearmint_april2013_mod.py | 5 +- HPOlib/tpecall.py | 1 - HPOlib/wrapping.py | 8 ++-- setup.py | 3 -- 19 files changed, 107 insertions(+), 54 deletions(-) diff --git a/HPOlib/Experiment.py b/HPOlib/Experiment.py index 0f9cd3a2..b246176c 100644 --- a/HPOlib/Experiment.py +++ b/HPOlib/Experiment.py @@ -24,7 +24,8 @@ import numpy as np -import Locker +import HPOlib.Locker as Locker + __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" diff --git a/HPOlib/Locker.py b/HPOlib/Locker.py index 69de10de..f78e4d98 100755 --- a/HPOlib/Locker.py +++ b/HPOlib/Locker.py @@ -23,6 +23,7 @@ import sys import time + def safe_delete(filename): cmd = 'mv "%s" "%s.delete" && rm "%s.delete"' % (filename, filename, filename) diff --git a/HPOlib/Plotting/plot_param.py b/HPOlib/Plotting/plot_param.py index c6205322..3b6db386 100644 --- a/HPOlib/Plotting/plot_param.py +++ b/HPOlib/Plotting/plot_param.py @@ -6,7 +6,6 @@ import sys from matplotlib.pyplot import tight_layout, figure, subplots_adjust, subplot, savefig, show -from pylab import setp from matplotlib.gridspec import GridSpec import numpy as np diff --git a/HPOlib/__init__.py b/HPOlib/__init__.py index 24cd9aaa..8a99b1bb 100644 --- a/HPOlib/__init__.py +++ b/HPOlib/__init__.py @@ -1,2 +1,4 @@ __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] -__contact__ = "automl.org" \ No newline at end of file +__contact__ = "automl.org" + +__version__="0.0.1" \ No newline at end of file diff --git a/HPOlib/benchmarks/branin/branin.py b/HPOlib/benchmarks/branin/branin.py index 46deb67c..f3271bb6 100644 --- a/HPOlib/benchmarks/branin/branin.py +++ b/HPOlib/benchmarks/branin/branin.py @@ -16,16 +16,83 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import sys -from numpy import NaN, pi, cos, ndarray -import benchmark_util +from numpy import pi, cos, ndarray +import HPOlib.benchmark_util as benchmark_util __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" +def parse_cli(): + """ + Provide a generic command line interface for benchmarks. It will just parse + the command line according to simple rules and return two dictionaries, one + containing all arguments for the benchmark algorithm like dataset, + crossvalidation metadata etc. and the containing all learning algorithm + hyperparameters. + + Parsing rules: + - Arguments with two minus signs are treated as benchmark arguments, Xalues + are not allowed to start with a minus. The last argument must --params, + starting the hyperparameter arguments. + - All arguments after --params are treated as hyperparameters to the + learning algorithm. Every parameter name must start with one minus and must + have exactly one value which has to be given in single quotes. + + Example: + python neural_network.py --folds 10 --fold 1 --dataset convex --params + -depth '3' -n_hid_0 '1024' -n_hid_1 '1024' -n_hid_2 '1024' -lr '0.01' + """ + args = {} + parameters = {} + + cli_args = sys.argv + found_params = False + skip = True + for idx, arg in enumerate(cli_args): + if skip: + skip = False + continue + else: + skip = True + + if arg == "--params": + found_params = True + skip = False + + elif arg[0:2] == "--" and not found_params: + if cli_args[idx+1][0] == "-": + raise ValueError("Argument name is not allowed to have a " + "leading minus %s" % cli_args[idx + 1]) + args[cli_args[idx][2:]] = cli_args[idx+1] + + elif arg[0:2] == "--" and found_params: + raise ValueError("You are trying to specify an argument after the " + "--params argument. Please change the order.") + + elif arg[0] == "-" and arg[0:2] != "--" and found_params: + if cli_args[idx+1][0] == "-": + raise ValueError("Hyperparameter name is not allowed to have a " + "leading minus %s" % cli_args[idx + 1]) + parameters[cli_args[idx][1:]] = cli_args[idx+1] + + elif arg[0] == "-" and arg[0:2] != "--" and not found_params: + raise ValueError("You either try to use arguments with only one lea" + "ding minus or try to specify a hyperparameter bef" + "ore the --params argument.") + + elif not found_params: + raise ValueError("Illegal command line string, expected an argument" + " starting with -- but found %s" % (arg,)) + + else: + raise ValueError("Illegal command line string, expected a hyperpara" + "meter starting with - but found %s" % (arg,)) + + return args, parameters + def branin(params, **kwargs): -# branin function +# branin function # The number of variables n = 2. # constraints: # -5 <= x <= 10, 0 <= y <= 15 diff --git a/HPOlib/benchmarks/har6/har6.py b/HPOlib/benchmarks/har6/har6.py index ac0aa0c6..26371843 100644 --- a/HPOlib/benchmarks/har6/har6.py +++ b/HPOlib/benchmarks/har6/har6.py @@ -17,7 +17,8 @@ # along with this program. If not, see . from numpy import NaN, array, ndarray, exp -import benchmark_util + +import HPOlib.benchmark_util as benchmark_util __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index fb113091..789b4bf1 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -16,8 +16,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python - import os import subprocess import sys diff --git a/HPOlib/config_parser/parse.py b/HPOlib/config_parser/parse.py index 2d822d67..7bc693a7 100644 --- a/HPOlib/config_parser/parse.py +++ b/HPOlib/config_parser/parse.py @@ -16,8 +16,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python - import os import imp import sys diff --git a/HPOlib/cv.py b/HPOlib/cv.py index 69e86ce3..e397e9cc 100644 --- a/HPOlib/cv.py +++ b/HPOlib/cv.py @@ -16,21 +16,18 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python - -import cPickle import os import subprocess import sys import time -from importlib import import_module - import numpy as np from Experiment import Experiment -from wrapping_util import format_traceback -from wrapping_util import load_experiment_config_file + +from HPOlib.wrapping_util import format_traceback, load_experiment_config_file + + __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __license__ = "3-clause BSD License" diff --git a/HPOlib/hyperopt_august2013_mod.py b/HPOlib/hyperopt_august2013_mod.py index 4b81d9a5..dcd37c7b 100644 --- a/HPOlib/hyperopt_august2013_mod.py +++ b/HPOlib/hyperopt_august2013_mod.py @@ -16,13 +16,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python - import cPickle import os import sys -import wrapping_util +import HPOlib.wrapping_util as wrapping_util + version_info = ("# %76s #\n" % "https://github.com/hyperopt/hyperopt/tree/486aebec8a4170e4781d99bbd6cca09123b12717") @@ -31,6 +30,7 @@ optimizer_str = "hyperopt_august2013_mod" + def build_tpe_call(config, options, optimizer_dir): #For TPE we have to cd to the exp_dir call = "cd " + optimizer_dir + "\n" + \ @@ -86,9 +86,12 @@ def main(config, options, experiment_dir, **kwargs): import hyperopt version = SYSTEM_WIDE except ImportError: - cmd += "export PYTHONPATH=$PYTHONPATH:" + os.path.dirname(os.path.abspath(__file__)) + \ - "/optimizers/hyperopt_august2013_mod\n" - import optimizers.hyperopt_august2013_mod.hyperopt as hyperopt + try: + cmd += "export PYTHONPATH=$PYTHONPATH:" + os.path.dirname(os.path.abspath(__file__)) + \ + "/optimizers/hyperopt_august2013_mod\n" + import optimizers.hyperopt_august2013_mod.hyperopt as hyperopt + except ImportError, e: + import HPOlib.optimizers.hyperopt_august2013_mod.hyperopt as hyperopt version = AUGUST_2013_MOD path_to_optimizer = os.path.abspath(os.path.dirname(hyperopt.__file__)) diff --git a/HPOlib/randomtpe.py b/HPOlib/randomtpe.py index 3243ad10..6080bf83 100644 --- a/HPOlib/randomtpe.py +++ b/HPOlib/randomtpe.py @@ -21,8 +21,7 @@ import cPickle import os -from config_parser.parse import parse_config -import wrapping_util +import HPOlib.wrapping_util as wrapping_util __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" diff --git a/HPOlib/run_instance.py b/HPOlib/run_instance.py index 6238d7d2..80323f1f 100644 --- a/HPOlib/run_instance.py +++ b/HPOlib/run_instance.py @@ -16,7 +16,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python import cPickle import imp @@ -29,8 +28,8 @@ import numpy as np from config_parser.parse import parse_config -import Experiment -import wrapping_util +import HPOlib.Experiment as Experiment +import HPOlib.wrapping_util as wrapping_util __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" diff --git a/HPOlib/run_test.py b/HPOlib/run_test.py index d237f8b3..adb033d9 100644 --- a/HPOlib/run_test.py +++ b/HPOlib/run_test.py @@ -18,20 +18,15 @@ #!/usr/bin/env python import cPickle -from importlib import import_module from optparse import OptionParser import os -import re import subprocess -import sys -import time -from gzip import GzipFile as gfile import numpy as np from config_parser.parse import parse_config -from Experiment import Experiment -import wrapping_util +from HPOlib.Experiment import Experiment +import HPOlib.wrapping_util as wrapping_util __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" @@ -42,6 +37,7 @@ def loadExperimentFile(): experiment = Experiment(".", optimizer) return experiment + def main(): # Parse options and arguments usage = "Coming soon" diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 1bb70e94..65a53731 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -25,8 +25,8 @@ import time import subprocess -import Experiment -import wrapping_util +import HPOlib.Experiment as Experiment +import HPOlib.wrapping_util as wrapping_util __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" diff --git a/HPOlib/smac_2_06_01-dev.py b/HPOlib/smac_2_06_01-dev.py index 0ad317e5..750d1f1e 100644 --- a/HPOlib/smac_2_06_01-dev.py +++ b/HPOlib/smac_2_06_01-dev.py @@ -16,8 +16,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python - import glob import os import re @@ -26,7 +24,8 @@ import numpy as np -import wrapping_util +import HPOlib.wrapping_util as wrapping_util + __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" diff --git a/HPOlib/spearmint_april2013_mod.py b/HPOlib/spearmint_april2013_mod.py index 444c9e9b..70e34d6d 100644 --- a/HPOlib/spearmint_april2013_mod.py +++ b/HPOlib/spearmint_april2013_mod.py @@ -16,15 +16,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python - import cPickle import os import sys import numpy as np -import wrapping_util +import HPOlib.wrapping_util as wrapping_util + __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" diff --git a/HPOlib/tpecall.py b/HPOlib/tpecall.py index 02cbeb1e..945ecce7 100644 --- a/HPOlib/tpecall.py +++ b/HPOlib/tpecall.py @@ -23,7 +23,6 @@ from functools import partial from importlib import import_module import os -import numpy as np import hyperopt diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 3f0d4853..51bac864 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -16,10 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python - from argparse import ArgumentParser -from config_parser.parse import parse_config import imp import os import subprocess @@ -28,9 +25,10 @@ import numpy as np -import check_before_start -import Experiment +from config_parser.parse import parse_config +import HPOlib.check_before_start as check_before_start +import HPOlib.Experiment as Experiment __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" diff --git a/setup.py b/setup.py index c749a8ea..c65f0ffc 100644 --- a/setup.py +++ b/setup.py @@ -3,9 +3,7 @@ from setuptools.command.test import test as TestCommand from setuptools.command.install import install as InstallCommand import io -import codecs import os -import sys import HPOlib @@ -79,7 +77,6 @@ def run(self): author=HPOlib.__authors__, tests_require=['unittest'], install_requires=['argparse==1.2.1', - 'bson==0.3.3', 'matplotlib==1.3.1', 'networkx==1.8.1', 'numpy==1.8.0', From 6f723ea0d017b1fec3c75a58f94495a851b14bde Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 6 Feb 2014 16:05:13 +0100 Subject: [PATCH 018/139] FIX: HPOlib-run had shebang after the license --- HPOlib-run | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HPOlib-run b/HPOlib-run index 4e8ec3c3..eb5485b4 100755 --- a/HPOlib-run +++ b/HPOlib-run @@ -1,3 +1,5 @@ +#!/usr/bin/env python + ## # wrapping: A program making it easy to use hyperparameter # optimization software. @@ -16,8 +18,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python - __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" From 511b0764d87d45bb09707c5c221977222f24dda5 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Thu, 6 Feb 2014 16:23:57 +0100 Subject: [PATCH 019/139] FIX: Result is now only printed once fixes problem that result is printed twice which leads to problems with smac --- HPOlib/benchmarks/branin/branin.py | 69 ------------------------------ HPOlib/cv.py | 2 +- HPOlib/runsolver_wrapper.py | 2 +- 3 files changed, 2 insertions(+), 71 deletions(-) diff --git a/HPOlib/benchmarks/branin/branin.py b/HPOlib/benchmarks/branin/branin.py index 01f1bd9e..416c88fb 100644 --- a/HPOlib/benchmarks/branin/branin.py +++ b/HPOlib/benchmarks/branin/branin.py @@ -19,7 +19,6 @@ import HPOlib.benchmark_util as benchmark_util -import sys from numpy import pi, cos, ndarray import time @@ -27,74 +26,6 @@ __contact__ = "automl.org" -def parse_cli(): - """ - Provide a generic command line interface for benchmarks. It will just parse - the command line according to simple rules and return two dictionaries, one - containing all arguments for the benchmark algorithm like dataset, - crossvalidation metadata etc. and the containing all learning algorithm - hyperparameters. - - Parsing rules: - - Arguments with two minus signs are treated as benchmark arguments, Xalues - are not allowed to start with a minus. The last argument must --params, - starting the hyperparameter arguments. - - All arguments after --params are treated as hyperparameters to the - learning algorithm. Every parameter name must start with one minus and must - have exactly one value which has to be given in single quotes. - - Example: - python neural_network.py --folds 10 --fold 1 --dataset convex --params - -depth '3' -n_hid_0 '1024' -n_hid_1 '1024' -n_hid_2 '1024' -lr '0.01' - """ - args = {} - parameters = {} - - cli_args = sys.argv - found_params = False - skip = True - for idx, arg in enumerate(cli_args): - if skip: - skip = False - continue - else: - skip = True - - if arg == "--params": - found_params = True - skip = False - - elif arg[0:2] == "--" and not found_params: - if cli_args[idx+1][0] == "-": - raise ValueError("Argument name is not allowed to have a " - "leading minus %s" % cli_args[idx + 1]) - args[cli_args[idx][2:]] = cli_args[idx+1] - - elif arg[0:2] == "--" and found_params: - raise ValueError("You are trying to specify an argument after the " - "--params argument. Please change the order.") - - elif arg[0] == "-" and arg[0:2] != "--" and found_params: - if cli_args[idx+1][0] == "-": - raise ValueError("Hyperparameter name is not allowed to have a " - "leading minus %s" % cli_args[idx + 1]) - parameters[cli_args[idx][1:]] = cli_args[idx+1] - - elif arg[0] == "-" and arg[0:2] != "--" and not found_params: - raise ValueError("You either try to use arguments with only one lea" - "ding minus or try to specify a hyperparameter bef" - "ore the --params argument.") - - elif not found_params: - raise ValueError("Illegal command line string, expected an argument" - " starting with -- but found %s" % (arg,)) - - else: - raise ValueError("Illegal command line string, expected a hyperpara" - "meter starting with - but found %s" % (arg,)) - - return args, parameters - def branin(params, **kwargs): # branin function # The number of variables n = 2. diff --git a/HPOlib/cv.py b/HPOlib/cv.py index b93e0a78..2195a4ae 100644 --- a/HPOlib/cv.py +++ b/HPOlib/cv.py @@ -23,7 +23,7 @@ import numpy as np -from Experiment import Experiment +from HPOlib.Experiment import Experiment from HPOlib.wrapping_util import format_traceback, load_experiment_config_file diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 270c9b49..4bcc9cfc 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -269,7 +269,7 @@ def parse_output_files(cfg, run_instance_output, runsolver_output_file): os.remove(runsolver_output_file) rval = (cpu_time, wallclock_time, "SAT", float(result_array[6].strip(",")), cfg.get("DEFAULT", "function")) - print result_string + #print result_string else: rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("DEFAULT", From 7188f2c64d0629139f6b17e074935b6c02ef2a8d Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 10 Feb 2014 17:02:55 +0100 Subject: [PATCH 020/139] FIX: #23, requirements are checked before doing anything --- HPOlib/check_before_start.py | 8 +++++--- HPOlib/wrapping.py | 14 ++++++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index 789b4bf1..105722bb 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -20,6 +20,7 @@ import subprocess import sys import imp + from config_parser.parse import parse_config """This script checks whether all dependencies are installed""" @@ -122,18 +123,19 @@ def _check_function(experiment_dir, optimizer_dir): del fn -def _check_zeroth(experiment_dir): +def check_zeroth(experiment_dir): print "\tconfig.cfg..", + _check_modules() _check_config(experiment_dir) print "..passed" -def _check_first(experiment_dir): +def check_first(experiment_dir): """ Do some checks before optimizer is loaded """ main() -def _check_second(experiment_dir, optimizer_dir): +def check_second(experiment_dir, optimizer_dir): """ Do remaining tests """ print "\talgorithm..", _check_function(experiment_dir, optimizer_dir) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 51bac864..094e66a4 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -23,12 +23,9 @@ import sys import time -import numpy as np - from config_parser.parse import parse_config import HPOlib.check_before_start as check_before_start -import HPOlib.Experiment as Experiment __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" @@ -162,7 +159,12 @@ def main(): """Start an optimization of the HPOlib. For documentation see the comments inside this function and the general HPOlib documentation.""" experiment_dir = os.getcwd() - check_before_start._check_zeroth(experiment_dir) + check_before_start.check_zeroth(experiment_dir) + + # Now we can safely import non standard things + import numpy as np + import HPOlib.Experiment as Experiment + args, unknown = use_option_parser() optimizer = args.optimizer @@ -188,7 +190,7 @@ def main(): os.putenv('PATH', os.environ['PATH'] + ":" + wrapping_dir + "/../runsolver/src/") # _check_runsolver, _check_modules() - check_before_start._check_first(experiment_dir) + check_before_start.check_first(experiment_dir) # build call cmd = "export PYTHONPATH=$PYTHONPATH:" + wrapping_dir + "\n" @@ -213,7 +215,7 @@ def main(): save_config_to_file(f, config) # _check_function - check_before_start._check_second(experiment_dir, optimizer_dir) + check_before_start.check_second(experiment_dir, optimizer_dir) # initialize/reload pickle file if args.restore: From cd7ca1ab6cd07c290eef78076f98f71a98c222a9 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Tue, 11 Feb 2014 14:37:25 +0100 Subject: [PATCH 021/139] FIX: random crashes in Experiment.py and runsolver _wrapper.py due to the sanity check in line 331 which was not able to compare floats. runsolver_wrapper.py is now more robust, it checks whether the runtime appears at the beginning of a line (if not it doesn't parse it) --- HPOlib/Experiment.py | 4 +++- HPOlib/runsolver_wrapper.py | 4 ++-- HPOlib/wrapping_util.py | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/HPOlib/Experiment.py b/HPOlib/Experiment.py index b246176c..212cb2c9 100644 --- a/HPOlib/Experiment.py +++ b/HPOlib/Experiment.py @@ -25,6 +25,7 @@ import numpy as np import HPOlib.Locker as Locker +import HPOlib.wrapping_util as wrapping_util __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] @@ -327,7 +328,8 @@ def _sanity_check(self): # Backwards compability with numpy 1.6 wallclock_time = np.nansum(trial['instance_durations']) total_wallclock_time += wallclock_time if np.isfinite(wallclock_time) else 0 - assert (total_wallclock_time == self.total_wallclock_time), \ + assert (wrapping_util.float_eq(total_wallclock_time, + self.total_wallclock_time)), \ (total_wallclock_time, self.total_wallclock_time) # Automatically loads this object from a pickle file diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 270c9b49..4f3db044 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -108,9 +108,9 @@ def read_runsolver_output(runsolver_output_file): limit_exceeded = "Memory exceeded" if "Solver just ended. Dumping a history of the last" in line: solver_ended_section = True - if "Real time (s): " in line and solver_ended_section: + if re.search(r"Real time \(s\): ", line) and solver_ended_section: wallclock_time = float(line.split()[3]) - if "CPU time (s): " in line and solver_ended_section: + if re.search(r"^CPU time \(s\): ", line) and solver_ended_section: cpu_time = float(line.split()[3]) return cpu_time, wallclock_time, limit_exceeded diff --git a/HPOlib/wrapping_util.py b/HPOlib/wrapping_util.py index b3b78bcd..8541d5e3 100644 --- a/HPOlib/wrapping_util.py +++ b/HPOlib/wrapping_util.py @@ -36,8 +36,8 @@ def get_time_string(): return time_string -def float_eq( a, b, eps=0.0001 ): - return abs(math.log( a ) - math.log(b)) <= eps +def float_eq(a, b, eps=0.0001): + return abs(math.log(a+1) - math.log(b+1)) <= eps def format_traceback(exc_info): From 82b901ae39e4d5ae48a0fb09ce458b25df5edef6 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 17 Feb 2014 11:21:34 +0100 Subject: [PATCH 022/139] Adjust result.sh to match new folder names --- HPOlib/Plotting/plot_param.py | 1 + HPOlib/Plotting/results.sh | 246 +++++++++++++++++++--------------- 2 files changed, 136 insertions(+), 111 deletions(-) diff --git a/HPOlib/Plotting/plot_param.py b/HPOlib/Plotting/plot_param.py index 3b6db386..f54b051c 100644 --- a/HPOlib/Plotting/plot_param.py +++ b/HPOlib/Plotting/plot_param.py @@ -109,6 +109,7 @@ def plot_params(value_list, result_list, name, save="", title="", jitter=0): def main(): + usage = "python plot_param.py WhatIsThis * [options]" parser = optparse.OptionParser() parser.add_option("-s", "--save", dest="save", default="", help="Where to save plot instead of showing it?") diff --git a/HPOlib/Plotting/results.sh b/HPOlib/Plotting/results.sh index 3e93f392..7cef9f52 100644 --- a/HPOlib/Plotting/results.sh +++ b/HPOlib/Plotting/results.sh @@ -1,115 +1,139 @@ #!/bin/sh -echo "Looking for SMAC:" - -printf "%5s | %4s(%5s) | %5s | %10s | %10s\n" "Seed" "#run" "crash" "#iter" "Performance" "Test-Perf" -for i in `seq 1000 1000 10000` -do - directory=`ls | grep "^smac_${i}_"` - if [ -f "${directory}/smac.out" ] - then - it=`cat ${directory}/smac.out | grep "Model/Iteration used:" | tail -1` - it=`echo $it | cut -d' ' -f3` - - per=`cat ${directory}/smac.out | grep "Performance of the Incumbent:" | tail -1` - per=`echo $per | cut -d' ' -f5` - - num=`cat ${directory}/smac.out | grep "Algorithm Runs used:" | tail -1` - num=`echo $num | cut -d' ' -f4` - - numC=`ls ${directory}/ | grep 'instance.out$' | wc -l` - numC=$(($numC - 1)) - - test_error=999 - test_file=`ls "$directory/" | grep "_test_run.out"` - if [ -f "${directory}/$test_file" ] +directory=`ls | grep "^smac_2_06_01-dev\>"` +if [ -a "${directory}" ] then - test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1` - test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1` - fi - printf "%5s | %4s(%5s) | %5s | %10f | %10f\n" "$i" "$num" "$numC" "$it" "$per" "$test_error" - fi - -done - -echo "Looking for TPE:" - -printf "%5s | %4s(%5s) | %10s | %10s\n" "Seed" "#run" "crash" "Performance" "Test-Perf" -for i in `seq 1000 1000 10000` -do - directory=`ls | grep "^tpe_${i}_"` - if [ -a "${directory}/tpe.out" ] - then - num=`cat ${directory}/tpe.out | grep "Result:" | wc -l` - - per=`cat ${directory}/tpe.out | grep "Result:" | sort -r | tail -1` - per=`echo $per | cut -d' ' -f2` - - numC=`ls ${directory}/ | grep 'instance.out$' | wc -l` - numC=$(($numC - 1)) - - test_error=999 - test_file=`ls -1 "$directory" | grep "_test_run.out"` - if [ -f "$directory/$test_file" ] + echo "Looking for SMAC (smac_2_06_01-dev):" + + printf "%5s | %4s(%5s) | %5s | %10s | %10s\n" "Seed" "#run" "crash" "#iter" "Performance" "Test-Perf" + for i in `seq 1000 1000 10000` + do + directory=`ls | grep "^smac_2_06_01-dev_${i}_"` + if [ -f "${directory}/smac_2_06_01-dev.out" ] + then + it=`cat ${directory}/smac*.out | grep "Model/Iteration used:" | tail -1` + it=`echo $it | cut -d' ' -f3` + + per=`cat ${directory}/smac*.out | grep "Performance of the Incumbent:" | tail -1` + per=`echo $per | cut -d' ' -f5` + + num=`cat ${directory}/smac*.out | grep "Algorithm Runs used:" | tail -1` + num=`echo $num | cut -d' ' -f4` + + numC=`ls ${directory}/ | grep 'instance.out$' | wc -l` + numC=$(($numC - 1)) + + test_error=999 + test_file=`ls "$directory/" | grep "_test_run.out"` + if [ -f "${directory}/$test_file" ] + then + test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1` + test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1` + fi + printf "%5s | %4s(%5s) | %5s | %10f | %10f\n" "$i" "$num" "$numC" "$it" "$per" "$test_error" + fi + + done +fi + + +directory=`ls | grep "^smac-v2_06_02-partial38\>"` +if [ -a "${directory}" ] +then + + echo "Looking for SMAC (smac-v2_06_02-partial38):" + + printf "%5s | %4s(%5s) | %5s | %10s | %10s\n" "Seed" "#run" "crash" "#iter" "Performance" "Test-Perf" + for i in `seq 1000 1000 10000` + do + directory=`ls | grep "^smac-v2_06_02-partial38_${i}_"` + if [ -f "${directory}/smac-v2_06_02-partial38.out" ] + then + it=`cat ${directory}/smac*.out | grep "Model/Iteration used:" | tail -1` + it=`echo $it | cut -d' ' -f3` + + per=`cat ${directory}/smac*.out | grep "Performance of the Incumbent:" | tail -1` + per=`echo $per | cut -d' ' -f5` + + num=`cat ${directory}/smac*.out | grep "Algorithm Runs used:" | tail -1` + num=`echo $num | cut -d' ' -f4` + + numC=`ls ${directory}/ | grep 'instance.out$' | wc -l` + numC=$(($numC - 1)) + + test_error=999 + test_file=`ls "$directory/" | grep "_test_run.out"` + if [ -f "${directory}/$test_file" ] + then + test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1` + test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1` + fi + printf "%5s | %4s(%5s) | %5s | %10f | %10f\n" "$i" "$num" "$numC" "$it" "$per" "$test_error" + fi + + done +fi + + +directory=`ls | grep "^hyperopt_august2013_mod\>"` +if [ -a "${directory}" ] +then + echo "Looking for TPE:" + + printf "%5s | %4s(%5s) | %10s | %10s\n" "Seed" "#run" "crash" "Performance" "Test-Perf" + for i in `seq 1000 1000 10000` + do + directory=`ls | grep "^tpe_${i}_"` + if [ -a "${directory}/tpe.out" ] + then + num=`cat ${directory}/tpe.out | grep "Result:" | wc -l` + + per=`cat ${directory}/tpe.out | grep "Result:" | sort -r | tail -1` + per=`echo $per | cut -d' ' -f2` + + numC=`ls ${directory}/ | grep 'instance.out$' | wc -l` + numC=$(($numC - 1)) + + test_error=999 + test_file=`ls -1 "$directory" | grep "_test_run.out"` + if [ -f "$directory/$test_file" ] + then + test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1` + test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1` + fi + printf "%5s | %4s(%5s) | %10f | %10f\n" "$i" "$num" "$numC" "$per" "$test_error" + fi + done +fi + + +directory=`ls | grep "^spearmint_april2013_mod\>"` +if [ -a "${directory}" ] then - test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1` - test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1` - fi - printf "%5s | %4s(%5s) | %10f | %10f\n" "$i" "$num" "$numC" "$per" "$test_error" - fi -done - -echo "Looking for Spearmint:" - -printf "%5s | %4s(%5s) | %10s | %10s\n" "Seed" "#run" "crash" "Performance" "Test-Perf" -for i in `seq 1000 1000 10000` -do - directory=`ls | grep "^spearmint_${i}_"` - if [ -a "${directory}/spearmint.out" ] - then - num=`cat ${directory}/spearmint.out | grep " pending " | tail -1` - num=`echo $num | cut -d' ' -f5` - per=`cat ${directory}/spearmint.out | grep "best:" | tail -1` - per=`echo $per | cut -d' ' -f3` - - numC=`ls ${directory}/ | grep 'instance.out$' | wc -l` - numC=$(($numC - 1)) - - test_error=999 - test_file=`ls -1 "$directory" | grep "_test_run.out"` - if [ -f "$directory/$test_file" ] - then - test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1` - test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1` - fi - printf "%5s | %4s(%5s) | %10f | %10f\n" "$i" "$num" "$numC" "$per" "$test_error" - fi -done - -echo "Looking for Random:" - -printf "%5s | %4s(%5s) | %10s | %10s\n" "Seed" "#run" "crash" "Performance" "Test-Perf" -for i in `seq 1000 1000 10000` -do - directory=`ls | grep "^randomtpe_${i}_"` - if [ -a "${directory}/randomtpe.out" ] - then - num=`cat ${directory}/randomtpe.out | grep "Result:" | wc -l` - - per=`cat ${directory}/randomtpe.out | grep "Result:" | sort -r | tail -1` - per=`echo $per | cut -d' ' -f2` - - numC=`ls ${directory}/ | grep 'instance.out$' | wc -l` - numC=$(($numC - 1)) - - test_error=999 - test_file=`ls -1 "$directory" | grep "_test_run.out"` - if [ -f "$directory/$test_file" ] - then - test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1` - test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1` - fi - printf "%5s | %4s(%5s) | %10f | %10f\n" "$i" "$num" "$numC" "$per" "$test_error" - fi -done - + echo "Looking for Spearmint:" + + printf "%5s | %4s(%5s) | %10s | %10s\n" "Seed" "#run" "crash" "Performance" "Test-Perf" + for i in `seq 1000 1000 10000` + do + directory=`ls | grep "^spearmint_${i}_"` + if [ -a "${directory}/spearmint.out" ] + then + num=`cat ${directory}/spearmint.out | grep " pending " | tail -1` + num=`echo $num | cut -d' ' -f5` + per=`cat ${directory}/spearmint.out | grep "best:" | tail -1` + per=`echo $per | cut -d' ' -f3` + + numC=`ls ${directory}/ | grep 'instance.out$' | wc -l` + numC=$(($numC - 1)) + + test_error=999 + test_file=`ls -1 "$directory" | grep "_test_run.out"` + if [ -f "$directory/$test_file" ] + then + test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1` + test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1` + fi + printf "%5s | %4s(%5s) | %10f | %10f\n" "$i" "$num" "$numC" "$per" "$test_error" + fi + done +fi From 2552057ed47af3633bc2206265fef675397369e2 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 18 Feb 2014 11:06:24 +0100 Subject: [PATCH 023/139] ENHANCEMENT: issue #18 and #4, installing runsolver setup.py: During installation, setup.py tries to set up runsolver as a script It * checks whether runsolver is already installed and copies this version (needs to be rethought) or * download and installs runsolver from http://www.cril.univ-artois.fr/~roussel/runsolver/runsolver-3.3.2.tar.bz2 or * if both fails, tells the user do download, install and add runsolver to $PATH plot_param.py: Minor changes to interface --- HPOlib/Plotting/plot_param.py | 4 +- README.txt => README | 0 setup.py | 166 ++++++++++++++++++++++++++-------- 3 files changed, 132 insertions(+), 38 deletions(-) rename README.txt => README (100%) diff --git a/HPOlib/Plotting/plot_param.py b/HPOlib/Plotting/plot_param.py index f54b051c..f915a913 100644 --- a/HPOlib/Plotting/plot_param.py +++ b/HPOlib/Plotting/plot_param.py @@ -77,7 +77,9 @@ def plot_params(value_list, result_list, name, save="", title="", jitter=0): best_result = min(result_list) for idx in range(len(result_list)): + # noinspection PyArgumentList result_list[idx] += (np.random.rand(1) - 0.5) * jitter + # noinspection PyArgumentList value_list[idx] += (np.random.rand(1) - 0.5) * jitter # Plot @@ -110,7 +112,7 @@ def plot_params(value_list, result_list, name, save="", title="", jitter=0): def main(): usage = "python plot_param.py WhatIsThis * [options]" - parser = optparse.OptionParser() + parser = optparse.OptionParser(usage) parser.add_option("-s", "--save", dest="save", default="", help="Where to save plot instead of showing it?") parser.add_option("-p", "--parameter", dest="param", default="", diff --git a/README.txt b/README similarity index 100% rename from README.txt rename to README diff --git a/setup.py b/setup.py index c65f0ffc..dc53738e 100644 --- a/setup.py +++ b/setup.py @@ -1,32 +1,19 @@ -from __future__ import print_function -from setuptools import setup, find_packages -from setuptools.command.test import test as TestCommand -from setuptools.command.install import install as InstallCommand -import io +import hashlib +from setuptools import setup +from setuptools.command.test import test as test_command +from setuptools.command.install import install as install_command +import shutil +import urllib import os +import subprocess +import tarfile import HPOlib here = os.path.abspath(os.path.dirname(__file__)) - desc = 'A software that can be used to evaluate either black boxes or hyperparameter optimization algorithms' -long_desc = "HPOlib is a hyperparameter optimization library. It provides a common interface to three state of \n", \ - "the art hyperparameter optimization packages: SMAC, spearmint and hyperopt. Moreover, the library \n", \ - "provides optimization benchmarks which can be used to compare different hyperparameter optimization\n", \ - "packages and to establish standard test routines for hyperparameter optimization packages. " keywords = 'hyperparameter optimization empirical evaluation black box' - -def get_find_packages(): - packages = ['HPOlib', - 'HPOlib.config_parser', - 'HPOlib.benchmarks', - 'HPOlib.optimizers', - 'HPOlib.benchmarks.branin', - 'HPOlib.benchmarks.har6'] - #find_packages('.', exclude=('*.optimizers.*',)) - return packages - package_dir = {'HPOlib': 'HPOlib', 'HPOlib.config_parser': 'HPOlib/config_parser', 'HPOlib.benchmarks': 'HPOlib/benchmarks', @@ -37,27 +24,128 @@ def get_find_packages(): 'HPOlib.benchmarks.branin': ['*.cfg', '*/params.txt', '*/space.py', '*/config.pb'], 'HPOlib.benchmarks.har6': ['*.cfg', '*/params.txt', '*/space.py', '*/config.pb'], 'HPOlib.config_parser': ['*.cfg']} -scripts = ['HPOlib-run'] +data_files = [] +scripts = ['HPOlib-run', 'runsolver/src/runsolver'] -def read(*filenames, **kwargs): - encoding = kwargs.get('encoding', 'utf-8') - sep = kwargs.get('sep', '\n') - buf = [] - for filename in filenames: - with io.open(filename, encoding=encoding) as f: - buf.append(f.read()) - return sep.join(buf) +def read(fname): + return open(os.path.join(os.path.dirname(__file__), fname)).read() -class InstallRunsolver(InstallCommand): +def get_find_packages(): + packages = ['HPOlib', + 'HPOlib.config_parser', + 'HPOlib.benchmarks', + 'HPOlib.optimizers', + 'HPOlib.benchmarks.branin', + 'HPOlib.benchmarks.har6'] + return packages + + +class InstallRunsolver(install_command): """Is supposed to install the runsolver later on""" + runsolver_needs_to_be_installed = False + download_url = 'http://www.cril.univ-artois.fr/~roussel/runsolver/runsolver-3.3.2.tar.bz2' + md5_should = '62d139a6d2b1b376eb9ed59ccb5d6726' + save_as = os.path.join(os.getcwd(), "runsolver-3.3.2.tar.bz2") + extract_as = os.getcwd() + + def _check_for_runsolver(self): + name = 'runsolver' + flags = os.X_OK + path = os.environ.get('PATH', None) + if path is None: + return False + for p in os.environ.get('PATH', '').split(os.pathsep): + p = os.path.join(p, name) + if os.access(p, flags): + print "Copy runsolver from %s to runsolver/src/runsolver" % p + shutil.copy(p, os.path.join(os.getcwd(), 'runsolver/src/runsolver')) + return p + return False + + def _download_runsolver(self): + try: + urllib.urlretrieve(self.download_url, filename=self.save_as) + except Exception, e: + print("Error downloading %s: %s" % (self.download_url, e)) + return False + md5 = hashlib.md5(open(self.save_as).read()).hexdigest() + if md5 != self.md5_should: + print "md5 checksum has changed: %s to %s" % (self.md5_should, md5) + return False + return True + + def _extract_runsolver(self): + if tarfile.is_tarfile(self.save_as): + try: + tfile = tarfile.open(self.save_as) + tfile.extractall(self.extract_as) + except Exception, e: + print("Error occurred during extraction: %s" % e) + return False + return True + return False + + def _build(self): + print("Building runsolver") + cur_pwd = os.getcwd() + os.chdir(os.path.join(self.extract_as, "runsolver", "src")) + try: + subprocess.check_call("make") + except subprocess.CalledProcessError, e: + print "Error during building runsolver: %s" % e + os.chdir(cur_pwd) + return False + os.chdir(cur_pwd) + return True + def run(self): - print("Hello User,\n" - "you need to install runsolver on your own :-)") + downloaded, extracted, built = (False, False, False) + runsolver_needs_to_be_installed = False + + print "Check whether you already have installed runsolver" + runsolver_path = self._check_for_runsolver() + if not runsolver_path: + print "'runsolver' not found, so we try to install it" + runsolver_needs_to_be_installed = True + + download_url = 'http://www.cril.univ-artois.fr/~roussel/runsolver/runsolver-3.3.2.tar.bz2' + save_as = os.path.join(os.getcwd(), "runsolver-3.3.2.tar.bz2") + extract_as = os.getcwd() + if runsolver_needs_to_be_installed: + print("Downloading runsolver from %s, saving to %s" % (download_url, save_as)) + downloaded = self._download_runsolver() + + if runsolver_needs_to_be_installed and downloaded: + print("Extracting runsolver to %s" % extract_as) + extracted = self._extract_runsolver() + + if runsolver_needs_to_be_installed and extracted: + print("Building runsolver") + built = self._build() + if not built: + print "Try a second time and replace '/usr/include/asm/unistd.h' with '/usr/include/unistd.h'" + call = "sed -i 's/\/usr\/include\/asm\/unistd/\/usr\/include\/unistd/g' runsolver/src/Makefile" + try: + subprocess.check_call(call, shell=True) + except subprocess.CalledProcessError, e: + print "Replacing did not work: %s" % e + built = self._build() + + install_command.run(self) + + # Give detailed output to user + if runsolver_needs_to_be_installed and not built: + print "[ERROR] Please install runsolver on your own! You can download it from:\n%s" % download_url + print "[ERROR] Be sure 'runsolver' can be found during runtime in your $PATH variable!" + if not runsolver_needs_to_be_installed and not built: + print "[INFO] 'runsolver' has been found on this system and was copied from: %s" % runsolver_path + if built: + print "'runsolver' has been downloaded and installed" -class PyTest(TestCommand): +class PyTest(test_command): def initialize_options(self): pass @@ -82,17 +170,21 @@ def run(self): 'numpy==1.8.0', 'protobuf==2.5.0', 'scipy==0.13.2', + 'pymongo==2.6.3', ], - cmdclass={'test': PyTest}, author_email='eggenspk@informatik.uni-freiburg.de', description=desc, - long_description=long_desc, + long_description=read("README"), keywords=keywords, packages=get_find_packages(), package_dir=package_dir, package_data=package_data, - #include_package_data=True, + data_files=data_files, scripts=scripts, + cmdclass={ + 'install': InstallRunsolver, + 'test': PyTest + }, classifiers=[ 'Programming Language :: Python :: 2.7', 'Development Status :: 2 - Pre-Alpha', From e3e2c8eac0fcf5ce37e8ab392b0e3b23ba609043 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 18 Feb 2014 11:14:04 +0100 Subject: [PATCH 024/139] FIX: Experiment.get_best() returns correct trial --- HPOlib/Experiment.py | 16 +++++++++------- HPOlib/wrapping.py | 8 +++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/HPOlib/Experiment.py b/HPOlib/Experiment.py index 212cb2c9..47af6519 100644 --- a/HPOlib/Experiment.py +++ b/HPOlib/Experiment.py @@ -162,18 +162,20 @@ def get_broken_jobs(self): # Get the best value so far def get_best(self): - best = 0 + best_idx = -1 + best_value = sys.maxint for i, trial in enumerate(self.trials): - res = np.NaN + tmp_res = np.NaN if trial['result'] == trial['result']: - res = trial['result'] + tmp_res = trial['result'] elif np.isfinite(trial['instance_results']).any(): - res = scipy.nanmean(trial['instance_results']) + tmp_res = scipy.nanmean(trial['instance_results']) else: continue - if res < self.trials[best]: - best = i - return self.trials[best] + if tmp_res < best_value: + best_idx = i + best_value = tmp_res + return self.trials[best_idx] def get_trial_from_id(self, _id): return self.trials[_id] diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 094e66a4..11b845e0 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -297,7 +297,6 @@ def main(): #noinspection PyProtectedMember trials._save_jobs() # trials.finish_experiment() - print "Finished with return code: " + str(ret) total_time = 0 print trials.get_best() #noinspection PyBroadException @@ -311,9 +310,12 @@ def main(): (calculate_wrapping_overhead(trials)) print "The algorithm itself took %f seconds" % \ trials.total_wallclock_time - except: - pass + except Exception, e: + print "Experiment itself went fine, " \ + "but calculating durations of optimization failed:\n%s" % e del trials + print "-----------------------END--------------------------------------" + print "Finished with return code: " + str(ret) return ret if __name__ == "__main__": From a614809882432977d08181ee93466744c39b3797 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 18 Feb 2014 14:05:10 +0100 Subject: [PATCH 025/139] ENHANCEMENT: #7 More optimization output stdout/stderr of subprocess.Popen is redirected. wrapping.py accepts 2 new arguments, if none is set the default behaviour is to only print stderr. Arguments are: --silent: Nothing is printed --verbose: stderr/stdout is printed All output is still written to a file. Minor Changes: Calculation of duration of HPOlib/optimizer/benchmark in wrapping.py. Fix imports and bugs, clarify exception and output. --- HPOlib/wrapping.py | 75 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 11b845e0..98206ce3 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -36,7 +36,12 @@ def calculate_wrapping_overhead(trials): for times in zip(trials.cv_starttime, trials.cv_endtime): wrapping_time += times[1] - times[0] - wrapping_time = wrapping_time - np.nansum(trials.instance_durations) + # We need to import numpy again + import numpy as np + benchmark_time = 0 + for t in trials.trials: + benchmark_time += np.nansum(t['instance_durations']) + wrapping_time = wrapping_time - benchmark_time return wrapping_time @@ -59,7 +64,9 @@ def calculate_optimizer_time(trials): optimizer_time.append(trials.endtime[time_idx] - trials.cv_endtime[-1]) trials.optimizer_time = optimizer_time - + + # We need to import numpy again + import numpy as np return np.nansum(optimizer_time) @@ -86,6 +93,11 @@ def use_option_parser(): restore_help_string = "Restore the state from a given directory" parser.add_argument("--restore", default=None, dest="restore", help=restore_help_string) + parser.add_argument("--silent", default=False, action="store_true", + dest="silent", help="Don't print the optimizer output") + parser.add_argument("--verbose", default=False, action="store_true", + dest="verbose", + help="Print stderr/stdout for optimizer well, overrides --silent") args, unknown = parser.parse_known_args() return args, unknown @@ -288,34 +300,69 @@ def main(): print cmd output_file = os.path.join(optimizer_dir, optimizer + ".out") fh = open(output_file, "a") - process = subprocess.Popen(cmd, stdout=fh, stderr=fh, shell=True, - executable="/bin/bash") + # process = subprocess.Popen(cmd, stdout=fh, stderr=fh, shell=True, executable="/bin/bash") + # ret = process.wait() + print "-----------------------RUNNING----------------------------------" - ret = process.wait() + + if args.verbose: + # Print std and err output for optimizer + proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + print 'Optimizer runs with PID (+1):', proc.pid + + while proc.poll() is None: + next_line = proc.stdout.readline() + fh.write(next_line) + sys.stdout.write(next_line) + sys.stdout.flush() + + elif args.silent: + # Print nothing + proc = subprocess.Popen(cmd, shell=True, stdin=fh, stdout=fh, stderr=fh) + print 'Optimizer runs with PID (+1):', proc.pid + + else: + # Print only stderr + proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=fh, stderr=subprocess.PIPE) + print 'Optimizer runs with PID (+1):', proc.pid + + while proc.poll() is None: + next_line = proc.stderr.readline() + fh.write(next_line) + sys.stdout.write("[ERR]:" + next_line) + sys.stdout.flush() + + ret = proc.returncode + + print "-----------------------END--------------------------------------" + trials = Experiment.Experiment(optimizer_dir, optimizer) trials.endtime.append(time.time()) #noinspection PyProtectedMember trials._save_jobs() # trials.finish_experiment() total_time = 0 + print "### Best result" print trials.get_best() - #noinspection PyBroadException + print "###\n### Durations" try: for starttime, endtime in zip(trials.starttime, trials.endtime): total_time += endtime - starttime print "Needed a total of %f seconds" % total_time - print "The optimizer %s took %f seconds" % \ - (optimizer, calculate_optimizer_time(trials)) - print "The overhead of the wrapping software is %f seconds" % \ + print "The optimizer %s took %10.5f seconds" %\ + (optimizer, float(calculate_optimizer_time(trials))) + print "The overhead of HPOlib is %f seconds" % \ (calculate_wrapping_overhead(trials)) - print "The algorithm itself took %f seconds" % \ + print "The benchmark itself took %f seconds" % \ trials.total_wallclock_time - except Exception, e: + except Exception as e: + import HPOlib.wrapping_util + print HPOlib.wrapping_util.format_traceback(sys.exc_info()) print "Experiment itself went fine, " \ - "but calculating durations of optimization failed:\n%s" % e + "but calculating durations of optimization failed:", sys.exc_info()[0], e del trials - print "-----------------------END--------------------------------------" - print "Finished with return code: " + str(ret) + print "###\nFinished with return code: " + str(ret) return ret if __name__ == "__main__": From d2705e482075e8e6b9d61c1f8c35a07eb2089886 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 18 Feb 2014 14:58:37 +0100 Subject: [PATCH 026/139] REFACTOR: Clean up argparser --- HPOlib/wrapping.py | 55 ++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 98206ce3..d3444530 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -25,6 +25,7 @@ from config_parser.parse import parse_config +import HPOlib import HPOlib.check_before_start as check_before_start __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] @@ -70,7 +71,7 @@ def calculate_optimizer_time(trials): return np.nansum(optimizer_time) -def use_option_parser(): +def use_arg_parser(): """Parse all options which can be handled by the wrapping script. Unknown arguments are ignored and returned as a list. It is useful to check this list in your program to handle typos etc. @@ -79,25 +80,36 @@ def use_option_parser(): a tuple. The first element is an argparse.Namespace object, the second a list with all unknown arguments. """ - parser = ArgumentParser(description="Perform an experiment with the " - "HPOlib") + description = "Perform an experiment with HPOlib. " \ + "Call this script from experiment directory (containing 'config.cfg')" + epilog = "Your are using HPOlib " + HPOlib.__version__ + prog = "path/from/Experiment/to/HPOlib/wrapping.py" + + parser = ArgumentParser(description=description, prog=prog, epilog=epilog) + parser.add_argument("-o", "--optimizer", action="store", type=str, + dest="optimizer", help="Specify the optimizer name.", required=True) - parser.add_argument("-p", "--print", action="store_true", dest="printcmd", - default=False, + parser.add_argument("-p", "--print", action="store_true", + dest="printcmd", default=False, help="If set print the command instead of executing it") - parser.add_argument("-s", "--seed", dest="seed", default=1, type=int, + parser.add_argument("-s", "--seed", action="store", type=int, + dest="seed", default=1, help="Set the seed of the optimizer") - parser.add_argument("-t", "--title", dest="title", default=None, + parser.add_argument("-t", "--title", action="store", type=str, + dest="title", default=None, help="A title for the experiment") - restore_help_string = "Restore the state from a given directory" - parser.add_argument("--restore", default=None, dest="restore", - help=restore_help_string) - parser.add_argument("--silent", default=False, action="store_true", - dest="silent", help="Don't print the optimizer output") - parser.add_argument("--verbose", default=False, action="store_true", - dest="verbose", - help="Print stderr/stdout for optimizer well, overrides --silent") + parser.add_argument("-r", "--restore", action="store", type=str, + dest="restore", default=None, + help="Restore the state from a given directory") + + group = parser.add_mutually_exclusive_group() + group.add_argument("-q", "--silent", action="store_true", + dest="silent", default=False, + help="Don't print anything during optimization") + group.add_argument("-v", "--verbose", action="store_true", + dest="verbose", default=False, + help="Print stderr/stdout for optimizer") args, unknown = parser.parse_known_args() return args, unknown @@ -152,24 +164,26 @@ def override_config_with_cli_arguments(config, config_overrides): return config -def save_config_to_file(file, config): +def save_config_to_file(fn_handle, config): for section in config.sections(): - file.write("[DEFAULT]\n") + fn_handle.write("[DEFAULT]\n") for key in config.defaults(): if config.get(section, key) is None or \ config.get(section, key) != "": - file.write(key + " = " + config.get("DEFAULT", key) + "\n") + fn_handle.write(key + " = " + config.get("DEFAULT", key) + "\n") - file.write("[" + section + "]\n") + fn_handle.write("[" + section + "]\n") for key in config.options(section): if config.get(section, key) is None or \ config.get(section, key) != "": - file.write(key + " = " + config.get(section, key) + "\n") + fn_handle.write(key + " = " + config.get(section, key) + "\n") def main(): """Start an optimization of the HPOlib. For documentation see the comments inside this function and the general HPOlib documentation.""" + args, unknown = use_arg_parser() + experiment_dir = os.getcwd() check_before_start.check_zeroth(experiment_dir) @@ -177,7 +191,6 @@ def main(): import numpy as np import HPOlib.Experiment as Experiment - args, unknown = use_option_parser() optimizer = args.optimizer config_file = os.path.join(experiment_dir, "config.cfg") From 9c25da0eed566b65a8d17534d1d657657f0ab817 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 18 Feb 2014 15:01:17 +0100 Subject: [PATCH 027/139] Require python version 2.7 --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.py b/setup.py index dc53738e..523ff8c4 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,7 @@ import urllib import os import subprocess +import sys import tarfile import HPOlib @@ -101,6 +102,11 @@ def _build(self): return True def run(self): + + if sys.version_info < (2, 7, 0) or sys.version_info >= (2, 8, 0): + sys.stderr.write("HPOlib requires Python 2.7.0\n") + sys.exit(-1) + downloaded, extracted, built = (False, False, False) runsolver_needs_to_be_installed = False From 05d56a84910942930a24a64104198b3f5850a989 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Tue, 18 Feb 2014 15:25:35 +0100 Subject: [PATCH 028/139] ENHANCEMENT: Added some tests and a suite --- HPOlib/Experiment.py | 22 +- HPOlib/cv.py | 2 +- HPOlib/data_util.py | 21 +- tests/__init__.py | 0 tests/branin_test/__init__.py | 0 tests/branin_test/branin.py | 51 + tests/branin_test/config.cfg | 8 + tests/branin_test/params123 | 6 + tests/branin_test/random/space.py | 4 + tests/branin_test/smac/params.txt | 2 + tests/branin_test/spearmint/__init__.py | 0 tests/branin_test/spearmint/config.pb | 18 + tests/branin_test/tpe/space.py | 4 + tests/dummy_config.cfg | 4 + tests/run_instance_no_result.txt | 4 + tests/run_instance_result.txt | 1 + tests/runsolver_positive.txt | 69 + tests/runsolver_positive_with_warning.txt | 147 ++ tests/runsolver_vsize_exceeded.txt | 89 + tests/runsolver_wallclock_time_limit.txt | 89 + tests/search_spaces/__init__.py | 1 + tests/search_spaces/autoweka.py | 2028 +++++++++++++++++++++ tests/search_spaces/nips2011.py | 56 + tests/test_benchmark_util.py | 68 + tests/test_cv.py | 125 ++ tests/test_data_utils.py | 105 ++ tests/test_experiment.py | 237 +++ tests/test_get_trial_index.pkl | 421 +++++ tests/test_gridsearch.py | 67 + tests/test_runsolver_wrapper.py | 234 +++ tests/test_wrapping.py | 201 ++ tests/testsuite.py | 28 + 32 files changed, 4095 insertions(+), 17 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/branin_test/__init__.py create mode 100644 tests/branin_test/branin.py create mode 100644 tests/branin_test/config.cfg create mode 100644 tests/branin_test/params123 create mode 100644 tests/branin_test/random/space.py create mode 100644 tests/branin_test/smac/params.txt create mode 100644 tests/branin_test/spearmint/__init__.py create mode 100644 tests/branin_test/spearmint/config.pb create mode 100644 tests/branin_test/tpe/space.py create mode 100644 tests/dummy_config.cfg create mode 100644 tests/run_instance_no_result.txt create mode 100644 tests/run_instance_result.txt create mode 100644 tests/runsolver_positive.txt create mode 100644 tests/runsolver_positive_with_warning.txt create mode 100644 tests/runsolver_vsize_exceeded.txt create mode 100644 tests/runsolver_wallclock_time_limit.txt create mode 100644 tests/search_spaces/__init__.py create mode 100644 tests/search_spaces/autoweka.py create mode 100644 tests/search_spaces/nips2011.py create mode 100644 tests/test_benchmark_util.py create mode 100644 tests/test_cv.py create mode 100644 tests/test_data_utils.py create mode 100644 tests/test_experiment.py create mode 100644 tests/test_get_trial_index.pkl create mode 100644 tests/test_gridsearch.py create mode 100644 tests/test_runsolver_wrapper.py create mode 100644 tests/test_wrapping.py create mode 100644 tests/testsuite.py diff --git a/HPOlib/Experiment.py b/HPOlib/Experiment.py index 47af6519..b4b098a1 100644 --- a/HPOlib/Experiment.py +++ b/HPOlib/Experiment.py @@ -160,8 +160,13 @@ def get_complete_jobs(self): def get_broken_jobs(self): return np.nonzero(self.status_array() == BROKEN_STATE)[0] - # Get the best value so far - def get_best(self): + # Get the job id of the best value so far, if there is no result + # available, this method consults the instance_results. If there are more + # than one trials with the same response value, the first trial is + # considered to be the best. If no trial with a better response value + # than sys.maxint is found, a ValueError is raised. + # TODO: add a method that incomplete jobs are not considered + def get_arg_best(self): best_idx = -1 best_value = sys.maxint for i, trial in enumerate(self.trials): @@ -175,6 +180,13 @@ def get_best(self): if tmp_res < best_value: best_idx = i best_value = tmp_res + if best_idx == -1: + raise ValueError("No best value found.") + return best_idx + + # Get the best value so far, for more documentation see get_arg_best + def get_best(self): + best_idx = self.get_arg_best() return self.trials[best_idx] def get_trial_from_id(self, _id): @@ -258,13 +270,13 @@ def check_cv_finished(self, _id): # parameters. Useful to delete all unnecessary entries after a crash in order # to restart def remove_all_but_first_runs(self, restored_runs): - print "#########Restored runs", restored_runs - print self.instance_order, len(self.instance_order) + # print "#########Restored runs", restored_runs + # print self.instance_order, len(self.instance_order) if len(self.instance_order) == restored_runs: pass else: for _id, instance in self.instance_order[-1:restored_runs - 1:-1]: - print "Deleting", _id, instance + # print "Deleting", _id, instance if np.isfinite(self.trials[_id]['instance_durations'][instance]): self.total_wallclock_time -= \ self.trials[_id]['instance_durations'][instance] diff --git a/HPOlib/cv.py b/HPOlib/cv.py index 2195a4ae..e936b403 100644 --- a/HPOlib/cv.py +++ b/HPOlib/cv.py @@ -183,7 +183,7 @@ def __init__(self, param): new_dict[key] = value else: - raise Exception("Invalid params, cannot be flattened.") + raise Exception("Invalid params, cannot be flattened: \n%s." % params) params = new_dict return params diff --git a/HPOlib/data_util.py b/HPOlib/data_util.py index d47c1b31..3f494c5e 100644 --- a/HPOlib/data_util.py +++ b/HPOlib/data_util.py @@ -31,7 +31,7 @@ def load_file(filename, file_format, use_percentage): raise IOError("File %s not found", filename) if file_format == "gfile": - print "Loading file:", filename + # print "Loading file:", filename fh = gfile(filename, "rb") data = cPickle.load(fh) if use_percentage >= 100.: @@ -40,11 +40,10 @@ def load_file(filename, file_format, use_percentage): max_data = int(len(data) / 100. * use_percentage) data = data[:max_data] fh.close() - print "Done loading file:", filename, "has", len(data), "datapoints" - sys.stdout.flush() + # print "Done loading file:", filename, "has", len(data), "datapoints" elif file_format == "pickle": - print "Loading file:", filename + # print "Loading file:", filename fh = open(filename, "r") data = cPickle.load(fh) if use_percentage >= 100.: @@ -52,10 +51,10 @@ def load_file(filename, file_format, use_percentage): else: data = data[:len(data) / 100. * use_percentage] fh.close() - print "Done loading file:", filename, "has", len(data), "datapoints" + # print "Done loading file:", filename, "has", len(data), "datapoints" elif file_format == "numpy": - print "Loading file:", filename + # print "Loading file:", filename fh = open(filename, "r") data = np.load(fh) if use_percentage >= 100.: @@ -63,7 +62,7 @@ def load_file(filename, file_format, use_percentage): else: data = data[:len(data) / 100. * use_percentage] fh.close() - print "Done loading file:", filename, "has", len(data), "datapoints" + # print "Done loading file:", filename, "has", len(data), "datapoints" else: raise ValueError("%s is an unknown training_data_format", file_format) @@ -85,8 +84,8 @@ def custom_split(data, n_train, n_valid): "data (%d)") % (n_train, n_valid, len(data)) train = data[0:n_train] valid = data[n_train:] - print type(train) - print type(valid) + # print type(train) + # print type(valid) return train, valid @@ -107,7 +106,7 @@ def prepare_cv_for_fold(data, fold, folds): In case no data is handed over to the function, None is returned. """ - print "Fold", fold, "of ", folds, "folds" + # print "Fold", fold, "of ", folds, "folds" # Create an array with the split points if data is not None: data_len = len(data) @@ -133,7 +132,7 @@ def prepare_cv_for_fold(data, fold, folds): cv_split_mask[splits[i]:splits[i+1]] = 0 train = data[cv_split_mask] valid = data[~cv_split_mask] - print data.shape, train.shape, valid.shape, train.itemsize + # print data.shape, train.shape, valid.shape, train.itemsize else: train = [] valid = [] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/branin_test/__init__.py b/tests/branin_test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/branin_test/branin.py b/tests/branin_test/branin.py new file mode 100644 index 00000000..c1210a90 --- /dev/null +++ b/tests/branin_test/branin.py @@ -0,0 +1,51 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys +from numpy import NaN, pi, cos, ndarray + +def branin(params, **kwargs): +# branin function +# The number of variables n = 2. +# constraints: +# -5 <= x <= 10, 0 <= y <= 15 +# three global optima: (-pi, 12.275), (pi, 2.275), (9.42478, 2.475), where +# branin = 0.397887 + if "x" not in params or "y" not in params: + raise ValueError("No params found ['x', 'y']\n") + x = float(params["x"]) + y = float(params["y"]) + if type(x) == ndarray or type(y) == ndarray: + x = x[0] + y = y[0] + + if -5 > x or x > 10: + raise ValueError("X value not in between -5 and 10") + if 0 > y or y > 15: + raise ValueError("Y value not in between 0 and 15") + + result = (y-(5.1/(4*pi**2))*x**2+5*x/pi-6)**2 + result += 10*(1-1/(8*pi))*cos(x)+10 + return result + + +def main(params, **kwargs): + print 'Params: ', params, + y = branin(params, **kwargs) + print 'Result: ', y + return y diff --git a/tests/branin_test/config.cfg b/tests/branin_test/config.cfg new file mode 100644 index 00000000..6c6aef57 --- /dev/null +++ b/tests/branin_test/config.cfg @@ -0,0 +1,8 @@ +[SMAC] +p = smac/params.txt + +[DEFAULT] +function = ./branin.py +numberOfJobs = 2 +result_on_terminate = 1000 +numberCV = 5 diff --git a/tests/branin_test/params123 b/tests/branin_test/params123 new file mode 100644 index 00000000..c0076ed9 --- /dev/null +++ b/tests/branin_test/params123 @@ -0,0 +1,6 @@ +(dp1 +S'y' +S'5' +sS'x' +S'5' +s. \ No newline at end of file diff --git a/tests/branin_test/random/space.py b/tests/branin_test/random/space.py new file mode 100644 index 00000000..cccaea4d --- /dev/null +++ b/tests/branin_test/random/space.py @@ -0,0 +1,4 @@ +from hyperopt import hp + +space = {'x': hp.uniform('x', -5, 10), + 'y': hp.uniform('y', 0, 15) } \ No newline at end of file diff --git a/tests/branin_test/smac/params.txt b/tests/branin_test/smac/params.txt new file mode 100644 index 00000000..72b78373 --- /dev/null +++ b/tests/branin_test/smac/params.txt @@ -0,0 +1,2 @@ +x [-5, 10] [2.5] +y [0, 15] [7.5] diff --git a/tests/branin_test/spearmint/__init__.py b/tests/branin_test/spearmint/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/branin_test/spearmint/config.pb b/tests/branin_test/spearmint/config.pb new file mode 100644 index 00000000..70137ce6 --- /dev/null +++ b/tests/branin_test/spearmint/config.pb @@ -0,0 +1,18 @@ +language: PYTHON +name: "cv" + +variable { + name: "x" + type: FLOAT + size: 1 + min: -5 + max: 10 +} + +variable { + name: "y" + type: FLOAT + size: 1 + min: 0 + max: 15 +} diff --git a/tests/branin_test/tpe/space.py b/tests/branin_test/tpe/space.py new file mode 100644 index 00000000..cccaea4d --- /dev/null +++ b/tests/branin_test/tpe/space.py @@ -0,0 +1,4 @@ +from hyperopt import hp + +space = {'x': hp.uniform('x', -5, 10), + 'y': hp.uniform('y', 0, 15) } \ No newline at end of file diff --git a/tests/dummy_config.cfg b/tests/dummy_config.cfg new file mode 100644 index 00000000..a2bf4027 --- /dev/null +++ b/tests/dummy_config.cfg @@ -0,0 +1,4 @@ +[DEFAULT] +numberOfJobs = 1 +result_on_terminate = 1 +function = 1 \ No newline at end of file diff --git a/tests/run_instance_no_result.txt b/tests/run_instance_no_result.txt new file mode 100644 index 00000000..6134ff51 --- /dev/null +++ b/tests/run_instance_no_result.txt @@ -0,0 +1,4 @@ +0.05/0.17 Traceback (most recent call last): +0.05/0.17 File "/Software/HPOlib/HPOlib/benchmarks/branin/branin.py", line 21, in +0.05/0.17 import benchmark_util +0.05/0.17 ImportError: No module named benchmark_util \ No newline at end of file diff --git a/tests/run_instance_result.txt b/tests/run_instance_result.txt new file mode 100644 index 00000000..b36d9a5c --- /dev/null +++ b/tests/run_instance_result.txt @@ -0,0 +1 @@ +00.01/00.12 Result for ParamILS: SAT, 0.35, 1, 0.5, -1, Random file \ No newline at end of file diff --git a/tests/runsolver_positive.txt b/tests/runsolver_positive.txt new file mode 100644 index 00000000..0f869da8 --- /dev/null +++ b/tests/runsolver_positive.txt @@ -0,0 +1,69 @@ +runsolver Copyright (C) 2010-2013 Olivier ROUSSEL + +This is runsolver version 3.3.2 (svn: 1389) + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +command line: runsolver -o /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/branin/smac_2_06_01-dev_1_2014-2-4--13-34-14-689426/2014-2-4--13-37-6-214300_run_instance.out --timestamp --use-pty -W 3600 -C 14400 -M 2000 -d 0 python /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/branin/branin.py --fold 0 --folds 1 --params -x2.5 -y7.5 + +running on 4 cores: 0-3 + +Enforcing wall clock limit (soft limit, will send SIGTERM then SIGKILL): 3600 seconds +Enforcing CPUTime limit (soft limit, will send SIGTERM then SIGKILL): 14400 seconds +Enforcing CPUTime limit (hard limit, will send SIGXCPU): 14430 seconds +Enforcing VSIZE limit (soft limit, will send SIGTERM then SIGKILL): 2048000 KiB +Enforcing VSIZE limit (hard limit, stack expansion will fail with SIGSEGV, brk() and mmap() will return ENOMEM): 2099200 KiB +Using a pseudo terminal to collect output from the solver +Current StackSize limit: 8192 KiB + + +[startup+0 s] +/proc/loadavg: 0.93 0.47 0.34 3/498 26603 +/proc/meminfo: memFree=5018208/8174740 swapFree=0/0 +[pid=26603] ppid=26601 vsize=7764 CPUtime=0 cores=0-3 +/proc/26603/stat : 26603 (python) R 26601 26603 19549 34817 26580 4202496 591 0 0 0 0 0 0 0 20 0 1 0 60173368 7950336 402 4294967295 134512640 136958856 3216281168 3216280624 135835648 0 0 16781312 0 0 0 0 17 1 0 0 0 0 0 136965844 137310660 161521664 3216283729 3216283848 3216283848 3216285641 0 +/proc/26603/statm: 1941 402 305 598 0 159 0 + +Solver just ended. Dumping a history of the last processes samples + +[startup+0.200325 s] +/proc/loadavg: 0.93 0.47 0.34 3/498 26603 +/proc/meminfo: memFree=5018208/8174740 swapFree=0/0 +[pid=26603] ppid=26601 vsize=35076 CPUtime=0.13 cores=0-3 +/proc/26603/stat : 26603 (python) R 26601 26603 19549 34817 26580 4202496 2864 0 0 0 9 4 0 0 20 0 1 0 60173368 35917824 2147 4294967295 134512640 136958856 3216281168 3216245016 3078460452 0 0 16781312 2 0 0 0 17 0 0 0 0 0 0 136965844 137310660 161521664 3216283729 3216283848 3216283848 3216285641 0 +/proc/26603/statm: 8513 2172 909 598 0 1461 0 +Current children cumulated CPU time (s) 0.13 +Current children cumulated vsize (KiB) 35076 + +Child status: 1 +Real time (s): 0.259524 +CPU time (s): 0.188011 +CPU user time (s): 0.140008 +CPU system time (s): 0.048003 +CPU usage (%): 72.4446 +Max. virtual memory (cumulated for all children) (KiB): 35076 + +getrusage(RUSAGE_CHILDREN,...) data: +user time used= 0.140008 +system time used= 0.048003 +maximum resident set size= 11280 +integral shared memory size= 0 +integral unshared data size= 0 +integral unshared stack size= 0 +page reclaims= 3769 +page faults= 0 +swaps= 0 +block input operations= 0 +block output operations= 8 +messages sent= 0 +messages received= 0 +signals received= 0 +voluntary context switches= 206 +involuntary context switches= 31 + +runsolver used 0.012 second user time and 0.012 second system time + +The end \ No newline at end of file diff --git a/tests/runsolver_positive_with_warning.txt b/tests/runsolver_positive_with_warning.txt new file mode 100644 index 00000000..db1db64e --- /dev/null +++ b/tests/runsolver_positive_with_warning.txt @@ -0,0 +1,147 @@ +runsolver Copyright (C) 2010-2013 Olivier ROUSSEL + +This is runsolver version 3.3.2 (svn: 1389) + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +command line: runsolver -o /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/libsvm/gridsearch_1_2014-2-11--14-16-48-480307/2014-2-11--14-18-27-126358_run_instance.out --timestamp --use-pty -W 3600 -C 14400 -M 2000 -d 0 python /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/libsvm/libsvm.py --fold 8 --folds 10 --params -C -5 -gamma -12 + +running on 4 cores: 0-3 + +Enforcing wall clock limit (soft limit, will send SIGTERM then SIGKILL): 3600 seconds +Enforcing CPUTime limit (soft limit, will send SIGTERM then SIGKILL): 14400 seconds +Enforcing CPUTime limit (hard limit, will send SIGXCPU): 14430 seconds +Enforcing VSIZE limit (soft limit, will send SIGTERM then SIGKILL): 2048000 KiB +Enforcing VSIZE limit (hard limit, stack expansion will fail with SIGSEGV, brk() and mmap() will return ENOMEM): 2099200 KiB +Using a pseudo terminal to collect output from the solver +Current StackSize limit: 8192 KiB + + +[startup+0 s] +/proc/loadavg: 1.16 0.82 0.65 5/544 30014 +/proc/meminfo: memFree=3713800/8174740 swapFree=12/12 +[pid=30014] ppid=30012 vsize=7908 CPUtime=0 cores=0-3 +/proc/30014/stat : 30014 (python) R 30012 30014 26453 34818 28623 4202496 775 0 0 0 0 0 0 0 20 0 1 0 41695493 8097792 617 4294967295 134512640 136958856 3218443616 3218407092 3074162534 0 0 16781312 2 0 0 0 17 0 0 0 0 0 0 136965844 137310660 170483712 3218446272 3218446397 3218446397 3218448329 0 +/proc/30014/statm: 1977 617 421 598 0 195 0 + +[startup+0.062109 s] +/proc/loadavg: 1.16 0.82 0.65 5/544 30014 +/proc/meminfo: memFree=3713800/8174740 swapFree=12/12 +[pid=30014] ppid=30012 vsize=12236 CPUtime=0.03 cores=0-3 +/proc/30014/stat : 30014 (python) R 30012 30014 26453 34818 28623 4202496 1210 0 0 0 3 0 0 0 20 0 1 0 41695493 12529664 990 4294967295 134512640 136958856 3218443616 3218442812 135527088 0 0 16781312 2 0 0 0 17 0 0 0 0 0 0 136965844 137310660 170483712 3218446272 3218446397 3218446397 3218448329 0 +/proc/30014/statm: 3059 990 572 598 0 395 0 +Current children cumulated CPU time (s) 0.03 +Current children cumulated vsize (KiB) 12236 + +[startup+0.10047 s] +/proc/loadavg: 1.16 0.82 0.65 5/544 30014 +/proc/meminfo: memFree=3713800/8174740 swapFree=12/12 +[pid=30014] ppid=30012 vsize=14928 CPUtime=0.06 cores=0-3 +/proc/30014/stat : 30014 (python) R 30012 30014 26453 34818 28623 4202496 1599 0 0 0 5 1 0 0 20 0 1 0 41695493 15286272 1256 4294967295 134512640 136958856 3218443616 3218346768 3073911061 0 0 16781312 2 0 0 0 17 1 0 0 0 0 0 136965844 137310660 170483712 3218446272 3218446397 3218446397 3218448329 0 +/proc/30014/statm: 3732 1256 664 598 0 604 0 +Current children cumulated CPU time (s) 0.06 +Current children cumulated vsize (KiB) 14928 + +[startup+0.300223 s] +/proc/loadavg: 1.16 0.82 0.65 5/544 30014 +/proc/meminfo: memFree=3713800/8174740 swapFree=12/12 +[pid=30014] ppid=30012 vsize=36928 CPUtime=0.19 cores=0-3 +/proc/30014/stat : 30014 (python) D 30012 30014 26453 34818 28623 1077944320 3850 0 0 0 16 3 0 0 20 0 1 0 41695493 37814272 2845 4294967295 134512640 136958856 3218443616 3218380776 3077809188 0 0 16781312 2 4187962909 0 0 17 0 0 0 0 0 0 136965844 137310660 170483712 3218446272 3218446397 3218446397 3218448329 0 +/proc/30014/statm: 9232 2845 1034 598 0 2001 0 +Current children cumulated CPU time (s) 0.19 +Current children cumulated vsize (KiB) 36928 + +[startup+0.700156 s] +/proc/loadavg: 1.16 0.82 0.65 5/544 30014 +/proc/meminfo: memFree=3713800/8174740 swapFree=12/12 +[pid=30014] ppid=30012 vsize=58548 CPUtime=0.37 cores=0-3 +/proc/30014/stat : 30014 (python) D 30012 30014 26453 34818 28623 1077944320 7416 0 0 0 29 8 0 0 20 0 1 0 41695493 59953152 5552 4294967295 134512640 136958856 3218443616 3218376472 3077809188 0 0 16781312 2 4187962909 0 0 17 0 0 0 0 0 0 136965844 137310660 170483712 3218446272 3218446397 3218446397 3218448329 0 +/proc/30014/statm: 14637 5552 1786 598 0 4011 0 +Current children cumulated CPU time (s) 0.37 +Current children cumulated vsize (KiB) 58548 + +############ +# WARNING: +# current cumulated CPU time (0 s) is less than in the last sample (0.38 s) +# The time of a child was probably not reported to its father. +# (see the two samples below) +# Adding the difference (0.38 s) to the 'lost time'. + +[startup+0.900147 s] +/proc/loadavg: 1.16 0.82 0.65 5/544 30014 +/proc/meminfo: memFree=3713800/8174740 swapFree=12/12 +[pid=30014] ppid=30012 vsize=58944 CPUtime=0.38 cores=0-3 +/proc/30014/stat : 30014 (python) D 30012 30014 26453 34818 28623 1077944320 7538 0 0 0 30 8 0 0 20 0 1 0 41695493 60358656 5615 4294967295 134512640 136958856 3218443616 3218411784 3077809188 0 0 16781312 2 4187962909 0 0 17 0 0 0 0 0 0 136965844 137310660 170483712 3218446272 3218446397 3218446397 3218448329 0 +/proc/30014/statm: 14736 5615 1792 598 0 4091 0 +Current children cumulated CPU time (s) 0.38 +Current children cumulated vsize (KiB) 58944 + +[startup+1.00023 s] +/proc/loadavg: 1.16 0.82 0.65 3/546 30020 +/proc/meminfo: memFree=3725896/8174740 swapFree=12/12 +[pid=30014] ppid=-1 vsize=0 CPUtime=0 cores= +/proc/30014/stat : -- couldn't open stat file --/proc/30014/statm: Current children cumulated CPU time (s) 0 +Current children cumulated vsize (KiB) 0 +# +############ + + +Solver just ended. Dumping a history of the last processes samples + +[startup+0.900147 s] +/proc/loadavg: 1.16 0.82 0.65 5/544 30014 +/proc/meminfo: memFree=3713800/8174740 swapFree=12/12 +[pid=30014] ppid=30012 vsize=58944 CPUtime=0.38 cores=0-3 +/proc/30014/stat : 30014 (python) D 30012 30014 26453 34818 28623 1077944320 7538 0 0 0 30 8 0 0 20 0 1 0 41695493 60358656 5615 4294967295 134512640 136958856 3218443616 3218411784 3077809188 0 0 16781312 2 4187962909 0 0 17 0 0 0 0 0 0 136965844 137310660 170483712 3218446272 3218446397 3218446397 3218448329 0 +/proc/30014/statm: 14736 5615 1792 598 0 4091 0 +Current children cumulated CPU time (s) 0.38 +Current children cumulated vsize (KiB) 58944 + +[startup+1.00023 s] +/proc/loadavg: 1.16 0.82 0.65 3/546 30020 +/proc/meminfo: memFree=3725896/8174740 swapFree=12/12 +[pid=30014] ppid=-1 vsize=0 CPUtime=0 cores= +/proc/30014/stat : -- couldn't open stat file --/proc/30014/statm: Current children cumulated CPU time (s) 0 +Current children cumulated vsize (KiB) 0 + +Child status: 0 + +# WARNING: +# The CPU time of some children was not reported to their father +# (probably because of a missing or aborted wait()). +# This 'lost CPU time' is added to the watched process CPU time. +# lost CPU time (s): 0.38 +# lost CPU user time (s): 0.3 +# lost CPU system time (s): 0.08 + +Real time (s): 1.00203 +CPU time (s): 0.820027 +CPU user time (s): 0.644021 +CPU system time (s): 0.176006 +CPU usage (%): 81.8367 +Max. virtual memory (cumulated for all children) (KiB): 58944 + +getrusage(RUSAGE_CHILDREN,...) data: +user time used= 0.344021 +system time used= 0.096006 +maximum resident set size= 25684 +integral shared memory size= 0 +integral unshared data size= 0 +integral unshared stack size= 0 +page reclaims= 8607 +page faults= 0 +swaps= 0 +block input operations= 0 +block output operations= 8 +messages sent= 0 +messages received= 0 +signals received= 0 +voluntary context switches= 822 +involuntary context switches= 68 + +runsolver used 0 second user time and 0.028001 second system time + +The end \ No newline at end of file diff --git a/tests/runsolver_vsize_exceeded.txt b/tests/runsolver_vsize_exceeded.txt new file mode 100644 index 00000000..2e47069d --- /dev/null +++ b/tests/runsolver_vsize_exceeded.txt @@ -0,0 +1,89 @@ +runsolver Copyright (C) 2010-2013 Olivier ROUSSEL + +This is runsolver version 3.3.2 (svn: 1389) + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +command line: runsolver -o /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/branin/smac_2_06_01-dev_1_2014-2-4--13-34-14-689426/2014-2-4--13-59-11-643481_run_instance.out --timestamp --use-pty -W 3600 -C 14400 -M 10 -d 0 python /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/branin/branin.py --fold 0 --folds 1 --params -x2.5 -y7.5 + +running on 4 cores: 0-3 + +Enforcing wall clock limit (soft limit, will send SIGTERM then SIGKILL): 3600 seconds +Enforcing CPUTime limit (soft limit, will send SIGTERM then SIGKILL): 14400 seconds +Enforcing CPUTime limit (hard limit, will send SIGXCPU): 14430 seconds +Enforcing VSIZE limit (soft limit, will send SIGTERM then SIGKILL): 10240 KiB +Enforcing VSIZE limit (hard limit, stack expansion will fail with SIGSEGV, brk() and mmap() will return ENOMEM): 61440 KiB +Using a pseudo terminal to collect output from the solver +Current StackSize limit: 8192 KiB + + +[startup+0 s] +/proc/loadavg: 0.24 0.23 0.28 2/498 27516 +/proc/meminfo: memFree=4949692/8174740 swapFree=0/0 +[pid=27516] ppid=27514 vsize=7764 CPUtime=0 cores=0-3 +/proc/27516/stat : 27516 (python) D 27514 27516 19549 34817 27493 1077944320 720 0 0 0 0 0 0 0 20 0 1 0 60305913 7950336 527 4294967295 134512640 136958856 3213014688 3213000872 3078476836 0 0 16781312 2 4177571357 0 0 17 0 0 0 0 0 0 136965844 137310660 142368768 3213015121 3213015240 3213015240 3213017033 0 +/proc/27516/statm: 1941 527 369 598 0 159 0 + + + +Maximum VSize exceeded: sending SIGTERM then SIGKILL + +[startup+0.031982 s] +/proc/loadavg: 0.24 0.23 0.28 3/499 27517 +/proc/meminfo: memFree=4947708/8174740 swapFree=0/0 +[pid=27516] ppid=27514 vsize=14492 CPUtime=0.01 cores=0-3 +/proc/27516/stat : 27516 (python) D 27514 27516 19549 34817 27493 1077944320 1471 0 0 0 1 0 0 0 20 0 1 0 60305913 14839808 1138 4294967295 134512640 136958856 3213014688 3212920840 3078476836 0 0 16781312 2 4177571357 0 0 17 0 0 0 0 0 0 136965844 137310660 142368768 3213015121 3213015240 3213015240 3213017033 0 +/proc/27516/statm: 3623 1138 632 598 0 495 0 +Current children cumulated CPU time (s) 0.01 +Current children cumulated vsize (KiB) 14492 + +Sending SIGTERM to process tree (bottom up) + +[startup+0.031982 s] +/proc/loadavg: 0.24 0.23 0.28 3/499 27517 +/proc/meminfo: memFree=4947708/8174740 swapFree=0/0 +[pid=27516] ppid=27514 vsize=14492 CPUtime=0.01 cores=0-3 +/proc/27516/stat : 27516 (python) D 27514 27516 19549 34817 27493 1077944320 1471 0 0 0 1 0 0 0 20 0 1 0 60305913 14839808 1138 4294967295 134512640 136958856 3213014688 3212920840 3078476836 0 0 16781312 2 4177571357 0 0 17 0 0 0 0 0 0 136965844 137310660 142368768 3213015121 3213015240 3213015240 3213017033 0 +/proc/27516/statm: 3623 1138 632 598 0 495 0 +Current children cumulated CPU time (s) 0.01 +Current children cumulated vsize (KiB) 14492 +Sleeping 0 seconds + +Sending SIGKILL to process tree (bottom up) + +Sending SIGKILL to -27516 + +Solver just ended. Dumping a history of the last processes samples + +Child ended because it received signal 15 (SIGTERM) +Real time (s): 0.039276 +CPU time (s): 0.016 +CPU user time (s): 0.012 +CPU system time (s): 0.004 +CPU usage (%): 40.7373 +Max. virtual memory (cumulated for all children) (KiB): 14492 + +getrusage(RUSAGE_CHILDREN,...) data: +user time used= 0.012 +system time used= 0.004 +maximum resident set size= 4708 +integral shared memory size= 0 +integral unshared data size= 0 +integral unshared stack size= 0 +page reclaims= 1479 +page faults= 0 +swaps= 0 +block input operations= 0 +block output operations= 8 +messages sent= 0 +messages received= 0 +signals received= 0 +voluntary context switches= 62 +involuntary context switches= 4 + +runsolver used 0.004 second user time and 0.020001 second system time + +The end \ No newline at end of file diff --git a/tests/runsolver_wallclock_time_limit.txt b/tests/runsolver_wallclock_time_limit.txt new file mode 100644 index 00000000..5a21a1d5 --- /dev/null +++ b/tests/runsolver_wallclock_time_limit.txt @@ -0,0 +1,89 @@ +runsolver Copyright (C) 2010-2013 Olivier ROUSSEL + +This is runsolver version 3.3.2 (svn: 1389) + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +command line: runsolver -o /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/branin/smac_2_06_01-dev_1_2014-2-4--13-34-14-689426/2014-2-4--13-52-13-958676_run_instance.out --timestamp --use-pty -W -1 -C 14400 -M 2000 -d 0 python /home/feurerm/HPOlib/Software/HPOlib/HPOlib/benchmarks/branin/branin.py --fold 0 --folds 1 --params -x2.5 -y7.5 + +running on 4 cores: 0-3 + +Enforcing wall clock limit (soft limit, will send SIGTERM then SIGKILL): 4294967295 seconds +Enforcing CPUTime limit (soft limit, will send SIGTERM then SIGKILL): 14400 seconds +Enforcing CPUTime limit (hard limit, will send SIGXCPU): 14430 seconds +Enforcing VSIZE limit (soft limit, will send SIGTERM then SIGKILL): 2048000 KiB +Enforcing VSIZE limit (hard limit, stack expansion will fail with SIGSEGV, brk() and mmap() will return ENOMEM): 2099200 KiB +Using a pseudo terminal to collect output from the solver +Current StackSize limit: 8192 KiB + + +[startup+0 s] +/proc/loadavg: 0.29 0.34 0.33 3/497 27281 +/proc/meminfo: memFree=4970368/8174740 swapFree=0/0 +[pid=27281] ppid=27279 vsize=8168 CPUtime=0 cores=0-3 +/proc/27281/stat : 27281 (python) R 27279 27281 19549 34817 27258 4202496 796 0 0 0 0 0 0 0 20 0 1 0 60264144 8364032 617 4294967295 134512640 136958856 3214016992 3213989088 134704285 0 0 16781312 2 0 0 0 17 0 0 0 0 0 0 136965844 137310660 150605824 3214022737 3214022856 3214022856 3214024649 0 +/proc/27281/statm: 2042 617 421 598 0 260 0 + + + +Maximum wall clock time exceeded: sending SIGTERM then SIGKILL + +[startup+0.053088 s] +/proc/loadavg: 0.29 0.34 0.33 3/498 27282 +/proc/meminfo: memFree=4969376/8174740 swapFree=0/0 +[pid=27281] ppid=27279 vsize=13972 CPUtime=0.03 cores=0-3 +/proc/27281/stat : 27281 (python) R 27279 27281 19549 34817 27258 4202496 1425 0 0 0 2 1 0 0 20 0 1 0 60264144 14307328 1089 4294967295 134512640 136958856 3214016992 3213927008 3074546142 0 0 16781312 2 0 0 0 17 0 0 0 0 0 0 136965844 137310660 150605824 3214022737 3214022856 3214022856 3214024649 0 +/proc/27281/statm: 3493 1089 610 598 0 460 0 +Current children cumulated CPU time (s) 0.03 +Current children cumulated vsize (KiB) 13972 + +Sending SIGTERM to process tree (bottom up) + +[startup+0.053088 s] +/proc/loadavg: 0.29 0.34 0.33 3/498 27282 +/proc/meminfo: memFree=4969376/8174740 swapFree=0/0 +[pid=27281] ppid=27279 vsize=13972 CPUtime=0.03 cores=0-3 +/proc/27281/stat : 27281 (python) R 27279 27281 19549 34817 27258 4202496 1425 0 0 0 2 1 0 0 20 0 1 0 60264144 14307328 1089 4294967295 134512640 136958856 3214016992 3213927008 3074546142 0 0 16781312 2 0 0 0 17 0 0 0 0 0 0 136965844 137310660 150605824 3214022737 3214022856 3214022856 3214024649 0 +/proc/27281/statm: 3493 1089 610 598 0 460 0 +Current children cumulated CPU time (s) 0.03 +Current children cumulated vsize (KiB) 13972 +Sleeping 0 seconds + +Sending SIGKILL to process tree (bottom up) + +Sending SIGKILL to -27281 + +Solver just ended. Dumping a history of the last processes samples + +Child ended because it received signal 15 (SIGTERM) +Real time (s): 0.066825 +CPU time (s): 0.044002 +CPU user time (s): 0.028001 +CPU system time (s): 0.016001 +CPU usage (%): 65.8466 +Max. virtual memory (cumulated for all children) (KiB): 13972 + +getrusage(RUSAGE_CHILDREN,...) data: +user time used= 0.028001 +system time used= 0.016001 +maximum resident set size= 4556 +integral shared memory size= 0 +integral unshared data size= 0 +integral unshared stack size= 0 +page reclaims= 1440 +page faults= 0 +swaps= 0 +block input operations= 0 +block output operations= 8 +messages sent= 0 +messages received= 0 +signals received= 0 +voluntary context switches= 58 +involuntary context switches= 6 + +runsolver used 0.016001 second user time and 0.020001 second system time + +The end \ No newline at end of file diff --git a/tests/search_spaces/__init__.py b/tests/search_spaces/__init__.py new file mode 100644 index 00000000..8f0ce6cb --- /dev/null +++ b/tests/search_spaces/__init__.py @@ -0,0 +1 @@ +__author__ = 'feurerm' diff --git a/tests/search_spaces/autoweka.py b/tests/search_spaces/autoweka.py new file mode 100644 index 00000000..e6226f6e --- /dev/null +++ b/tests/search_spaces/autoweka.py @@ -0,0 +1,2028 @@ +from hyperopt import hp +import math + +space = ( + hp.choice('auto_param_2', [ + {'attributesearch':'NONE'}, + ( + hp.choice('auto_param_4', [ + { + 'attributesearch':'weka.attributeSelection.BestFirst', + 'assearch_wasbf__00__01_0_D':hp.choice('assearch_wasbf__00__01_0_D', ['2', '1', '0', ]), + 'assearch_wasbf__00__02_1_INT_N':hp.quniform('assearch_wasbf__00__02_1_INT_N', 2.0, 10.0,1.0), + 'assearch_wasbf__00__03_2_S':hp.choice('assearch_wasbf__00__03_2_S', ['0', ]), + }, + { + 'attributesearch':'weka.attributeSelection.GreedyStepwise', + 'assearch_wasgs__01__01_0_C':hp.choice('assearch_wasgs__01__01_0_C', ['REMOVE_PREV', 'REMOVED', ]), + 'assearch_wasgs__01__02_1_B':hp.choice('assearch_wasgs__01__02_1_B', ['REMOVE_PREV', 'REMOVED', ]), + 'assearch_wasgs__01__03_auto_param_7':hp.choice('assearch_wasgs__01__03_auto_param_7', [ + { + 'assearch_wasgs__01__03__00__00_2_R':'REMOVED', + 'assearch_wasgs__01__03__00__01_3_T':hp.uniform('assearch_wasgs__01__03__00__01_3_T', 0.0, 20.0), + }, + { + 'assearch_wasgs__01__03__01__00_2_R':'REMOVE_PREV', + 'assearch_wasgs__01__03__01__01_4_INT_N':hp.qloguniform('assearch_wasgs__01__03__01__01_4_INT_N', math.log(10.0), math.log(1000.0), 1.0), + }, + ]), + }, + { + 'attributesearch':'weka.attributeSelection.Ranker', + 'assearch_wasr__02__01_0_T':hp.uniform('assearch_wasr__02__01_0_T', 0.2, 10.0), + }, + ]), + hp.choice('auto_param_11', [ + { + 'attributeeval':'weka.attributeSelection.CfsSubsetEval', + 'aseval_wascse__00__01_0_M':hp.choice('aseval_wascse__00__01_0_M', ['REMOVE_PREV', 'REMOVED', ]), + 'aseval_wascse__00__02_1_L':hp.choice('aseval_wascse__00__02_1_L', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + 'attributeeval':'weka.attributeSelection.CorrelationAttributeEval', + }, + { + 'attributeeval':'weka.attributeSelection.GainRatioAttributeEval', + }, + { + 'attributeeval':'weka.attributeSelection.InfoGainAttributeEval', + 'aseval_wasigae__03__01_0_M':hp.choice('aseval_wasigae__03__01_0_M', ['REMOVE_PREV', 'REMOVED', ]), + 'aseval_wasigae__03__02_1_B':hp.choice('aseval_wasigae__03__02_1_B', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + 'attributeeval':'weka.attributeSelection.OneRAttributeEval', + 'aseval_wasorae__04__01_0_S':hp.choice('aseval_wasorae__04__01_0_S', ['0', ]), + 'aseval_wasorae__04__02_1_F':hp.quniform('aseval_wasorae__04__02_1_F', 2.0, 15.0,1.0), + 'aseval_wasorae__04__03_2_D':hp.choice('aseval_wasorae__04__03_2_D', ['REMOVE_PREV', 'REMOVED', ]), + 'aseval_wasorae__04__04_3_INT_B':hp.qloguniform('aseval_wasorae__04__04_3_INT_B', math.log(1.0), math.log(64.0), 1.0), + }, + { + 'attributeeval':'weka.attributeSelection.PrincipalComponents', + 'aseval_waspc__05__01_1_C':hp.choice('aseval_waspc__05__01_1_C', ['REMOVE_PREV', 'REMOVED', ]), + 'aseval_waspc__05__02_2_R':hp.uniform('aseval_waspc__05__02_2_R', 0.5, 1.0), + 'aseval_waspc__05__03_3_O':hp.choice('aseval_waspc__05__03_3_O', ['REMOVE_PREV', 'REMOVED', ]), + 'aseval_waspc__05__04_auto_param_18':hp.choice('aseval_waspc__05__04_auto_param_18', [ + { + 'aseval_waspc__05__04__00__00_num_HIDDEN':'0', + 'aseval_waspc__05__04__00__01_1_INT_A':hp.choice('aseval_waspc__05__04__00__01_1_INT_A', ['-1', ]), + }, + { + 'aseval_waspc__05__04__01__00_num_HIDDEN':'1', + 'aseval_waspc__05__04__01__01_2_INT_A':hp.qloguniform('aseval_waspc__05__04__01__01_2_INT_A', math.log(1.0), math.log(1024.0), 1.0), + }, + ]), + }, + { + 'attributeeval':'weka.attributeSelection.ReliefFAttributeEval', + 'aseval_wasrfae__06__01_0_INT_K':hp.qloguniform('aseval_wasrfae__06__01_0_INT_K', math.log(2.0), math.log(64.0), 1.0), + 'aseval_wasrfae__06__02_auto_param_22':hp.choice('aseval_wasrfae__06__02_auto_param_22', [ + { + 'aseval_wasrfae__06__02__00__00_1_W':'REMOVED', + 'aseval_wasrfae__06__02__00__01_2_INT_A':hp.qloguniform('aseval_wasrfae__06__02__00__01_2_INT_A', math.log(1.0), math.log(8.0), 1.0), + }, + { + 'aseval_wasrfae__06__02__01__00_1_W':'REMOVE_PREV', + }, + ]), + }, + { + 'attributeeval':'weka.attributeSelection.SymmetricalUncertAttributeEval', + 'aseval_wassuae__07__01_0_M':hp.choice('aseval_wassuae__07__01_0_M', ['REMOVE_PREV', 'REMOVED', ]), + }, + ]), + {'attributetime':'900.0'}, + ), + ]), + hp.choice('is_base', [ + hp.choice('base_choice', [ + { + 'targetclass':'weka.classifiers.bayes.BayesNet', + '_0_wcbbn__00__01_D':hp.choice('_0_wcbbn__00__01_D', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcbbn__00__02_Q':hp.choice('_0_wcbbn__00__02_Q', ['weka.classifiers.bayes.net.search.local.TAN', 'weka.classifiers.bayes.net.search.local.HillClimber', 'weka.classifiers.bayes.net.search.local.LAGDHillClimber', 'weka.classifiers.bayes.net.search.local.SimulatedAnnealing', 'weka.classifiers.bayes.net.search.local.TabuSearch', 'weka.classifiers.bayes.net.search.local.K2', ]), + }, + { + 'targetclass':'weka.classifiers.bayes.NaiveBayes', + '_0_wcbnb__01__01_K':hp.choice('_0_wcbnb__01__01_K', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcbnb__01__02_D':hp.choice('_0_wcbnb__01__02_D', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + 'targetclass':'weka.classifiers.bayes.NaiveBayesMultinomial', + }, + { + 'targetclass':'weka.classifiers.functions.Logistic', + '_0_wcfl__03__01_R':hp.loguniform('_0_wcfl__03__01_R', math.log(1.0E-12), math.log(10.0)), + }, + { + 'targetclass':'weka.classifiers.functions.MultilayerPerceptron', + '_0_wcfmp__04__01_L':hp.uniform('_0_wcfmp__04__01_L', 0.1, 1.0), + '_0_wcfmp__04__02_M':hp.uniform('_0_wcfmp__04__02_M', 0.1, 1.0), + '_0_wcfmp__04__03_B':hp.choice('_0_wcfmp__04__03_B', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcfmp__04__04_H':hp.choice('_0_wcfmp__04__04_H', ['t', 'i', 'o', 'a', ]), + '_0_wcfmp__04__05_C':hp.choice('_0_wcfmp__04__05_C', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcfmp__04__06_R':hp.choice('_0_wcfmp__04__06_R', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcfmp__04__07_D':hp.choice('_0_wcfmp__04__07_D', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcfmp__04__08_S':hp.choice('_0_wcfmp__04__08_S', ['1', ]), + }, + { + 'targetclass':'weka.classifiers.functions.SGD', + '_0_wcfsgd__05__01_F':hp.choice('_0_wcfsgd__05__01_F', ['2', '1', '0', ]), + '_0_wcfsgd__05__02_L':hp.loguniform('_0_wcfsgd__05__02_L', math.log(1.0E-5), math.log(0.1)), + '_0_wcfsgd__05__03_R':hp.loguniform('_0_wcfsgd__05__03_R', math.log(1.0E-12), math.log(10.0)), + '_0_wcfsgd__05__04_N':hp.choice('_0_wcfsgd__05__04_N', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcfsgd__05__05_M':hp.choice('_0_wcfsgd__05__05_M', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + 'targetclass':'weka.classifiers.functions.SMO', + '_0_wcfsmo__06__01_0_C':hp.uniform('_0_wcfsmo__06__01_0_C', 0.5, 1.5), + '_0_wcfsmo__06__02_1_N':hp.choice('_0_wcfsmo__06__02_1_N', ['2', '1', '0', ]), + '_0_wcfsmo__06__03_2_M':hp.choice('_0_wcfsmo__06__03_2_M', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcfsmo__06__04_5_QUOTE_END':hp.choice('_0_wcfsmo__06__04_5_QUOTE_END', ['REMOVED', ]), + '_0_wcfsmo__06__05_auto_param_33':hp.choice('_0_wcfsmo__06__05_auto_param_33', [ + { + '_0_wcfsmo__06__05__00__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.NormalizedPolyKernel', + '_0_wcfsmo__06__05__00__01_4_npoly_E':hp.uniform('_0_wcfsmo__06__05__00__01_4_npoly_E', 0.2, 5.0), + '_0_wcfsmo__06__05__00__02_4_npoly_L':hp.choice('_0_wcfsmo__06__05__00__02_4_npoly_L', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_0_wcfsmo__06__05__01__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.PolyKernel', + '_0_wcfsmo__06__05__01__01_4_poly_E':hp.uniform('_0_wcfsmo__06__05__01__01_4_poly_E', 0.2, 5.0), + '_0_wcfsmo__06__05__01__02_4_poly_L':hp.choice('_0_wcfsmo__06__05__01__02_4_poly_L', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_0_wcfsmo__06__05__02__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.Puk', + '_0_wcfsmo__06__05__02__01_4_puk_S':hp.uniform('_0_wcfsmo__06__05__02__01_4_puk_S', 0.1, 10.0), + '_0_wcfsmo__06__05__02__02_4_puk_O':hp.uniform('_0_wcfsmo__06__05__02__02_4_puk_O', 0.1, 1.0), + }, + { + '_0_wcfsmo__06__05__03__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.RBFKernel', + '_0_wcfsmo__06__05__03__01_4_rbf_G':hp.loguniform('_0_wcfsmo__06__05__03__01_4_rbf_G', math.log(1.0E-4), math.log(1.0)), + }, + ]), + }, + { + 'targetclass':'weka.classifiers.functions.SimpleLogistic', + '_0_wcfsl__07__01_S':hp.choice('_0_wcfsl__07__01_S', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcfsl__07__02_A':hp.choice('_0_wcfsl__07__02_A', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcfsl__07__03_auto_param_39':hp.choice('_0_wcfsl__07__03_auto_param_39', [ + { + '_0_wcfsl__07__03__00__00_W_HIDDEN':'0', + '_0_wcfsl__07__03__00__01_1_W':hp.choice('_0_wcfsl__07__03__00__01_1_W', ['0', ]), + }, + { + '_0_wcfsl__07__03__01__00_W_HIDDEN':'1', + '_0_wcfsl__07__03__01__01_2_W':hp.uniform('_0_wcfsl__07__03__01__01_2_W', 0.0, 1.0), + }, + ]), + }, + { + 'targetclass':'weka.classifiers.functions.VotedPerceptron', + '_0_wcfvp__08__01_INT_I':hp.quniform('_0_wcfvp__08__01_INT_I', 1.0, 10.0,1.0), + '_0_wcfvp__08__02_INT_M':hp.qloguniform('_0_wcfvp__08__02_INT_M', math.log(5000.0), math.log(50000.0), 1.0), + '_0_wcfvp__08__03_E':hp.uniform('_0_wcfvp__08__03_E', 0.2, 5.0), + }, + { + 'targetclass':'weka.classifiers.lazy.IBk', + '_0_wclib__09__01_E':hp.choice('_0_wclib__09__01_E', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wclib__09__02_INT_K':hp.qloguniform('_0_wclib__09__02_INT_K', math.log(1.0), math.log(64.0), 1.0), + '_0_wclib__09__03_X':hp.choice('_0_wclib__09__03_X', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wclib__09__04_F':hp.choice('_0_wclib__09__04_F', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wclib__09__05_I':hp.choice('_0_wclib__09__05_I', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + 'targetclass':'weka.classifiers.lazy.KStar', + '_0_wclks__10__01_INT_B':hp.quniform('_0_wclks__10__01_INT_B', 1.0, 100.0,1.0), + '_0_wclks__10__02_E':hp.choice('_0_wclks__10__02_E', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wclks__10__03_M':hp.choice('_0_wclks__10__03_M', ['n', 'd', 'm', 'a', ]), + }, + { + 'targetclass':'weka.classifiers.rules.DecisionTable', + '_0_wcrdt__11__01_E':hp.choice('_0_wcrdt__11__01_E', ['auc', 'rmse', 'mae', 'acc', ]), + '_0_wcrdt__11__02_I':hp.choice('_0_wcrdt__11__02_I', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcrdt__11__03_S':hp.choice('_0_wcrdt__11__03_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]), + '_0_wcrdt__11__04_X':hp.choice('_0_wcrdt__11__04_X', ['4', '2', '3', '1', ]), + }, + { + 'targetclass':'weka.classifiers.rules.JRip', + '_0_wcrjr__12__01_N':hp.uniform('_0_wcrjr__12__01_N', 1.0, 5.0), + '_0_wcrjr__12__02_E':hp.choice('_0_wcrjr__12__02_E', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcrjr__12__03_P':hp.choice('_0_wcrjr__12__03_P', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcrjr__12__04_INT_O':hp.quniform('_0_wcrjr__12__04_INT_O', 1.0, 5.0,1.0), + }, + { + 'targetclass':'weka.classifiers.rules.OneR', + '_0_wcror__13__01_INT_B':hp.qloguniform('_0_wcror__13__01_INT_B', math.log(1.0), math.log(32.0), 1.0), + }, + { + 'targetclass':'weka.classifiers.rules.PART', + '_0_wcrpart__14__01_INT_N':hp.quniform('_0_wcrpart__14__01_INT_N', 2.0, 5.0,1.0), + '_0_wcrpart__14__02_INT_M':hp.qloguniform('_0_wcrpart__14__02_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_0_wcrpart__14__03_R':hp.choice('_0_wcrpart__14__03_R', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcrpart__14__04_B':hp.choice('_0_wcrpart__14__04_B', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + 'targetclass':'weka.classifiers.rules.ZeroR', + }, + { + 'targetclass':'weka.classifiers.trees.DecisionStump', + }, + { + 'targetclass':'weka.classifiers.trees.J48', + '_0_wctj__17__01_O':hp.choice('_0_wctj__17__01_O', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wctj__17__02_U':hp.choice('_0_wctj__17__02_U', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wctj__17__03_B':hp.choice('_0_wctj__17__03_B', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wctj__17__04_J':hp.choice('_0_wctj__17__04_J', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wctj__17__05_A':hp.choice('_0_wctj__17__05_A', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wctj__17__06_S':hp.choice('_0_wctj__17__06_S', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wctj__17__07_INT_M':hp.qloguniform('_0_wctj__17__07_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_0_wctj__17__08_C':hp.uniform('_0_wctj__17__08_C', 0.0, 1.0), + }, + { + 'targetclass':'weka.classifiers.trees.LMT', + '_0_wctlmt__18__01_B':hp.choice('_0_wctlmt__18__01_B', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wctlmt__18__02_R':hp.choice('_0_wctlmt__18__02_R', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wctlmt__18__03_C':hp.choice('_0_wctlmt__18__03_C', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wctlmt__18__04_P':hp.choice('_0_wctlmt__18__04_P', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wctlmt__18__05_INT_M':hp.qloguniform('_0_wctlmt__18__05_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_0_wctlmt__18__06_A':hp.choice('_0_wctlmt__18__06_A', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wctlmt__18__07_auto_param_53':hp.choice('_0_wctlmt__18__07_auto_param_53', [ + { + '_0_wctlmt__18__07__00__00_W_HIDDEN':'0', + '_0_wctlmt__18__07__00__01_1_W':hp.choice('_0_wctlmt__18__07__00__01_1_W', ['0', ]), + }, + { + '_0_wctlmt__18__07__01__00_W_HIDDEN':'1', + '_0_wctlmt__18__07__01__01_2_W':hp.uniform('_0_wctlmt__18__07__01__01_2_W', 0.0, 1.0), + }, + ]), + }, + { + 'targetclass':'weka.classifiers.trees.REPTree', + '_0_wctrept__19__01_INT_M':hp.qloguniform('_0_wctrept__19__01_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_0_wctrept__19__02_V':hp.loguniform('_0_wctrept__19__02_V', math.log(1.0E-5), math.log(0.1)), + '_0_wctrept__19__03_P':hp.choice('_0_wctrept__19__03_P', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wctrept__19__04_auto_param_57':hp.choice('_0_wctrept__19__04_auto_param_57', [ + { + '_0_wctrept__19__04__00__00_depth_HIDDEN':'0', + '_0_wctrept__19__04__00__01_1_INT_L':hp.choice('_0_wctrept__19__04__00__01_1_INT_L', ['-1', ]), + }, + { + '_0_wctrept__19__04__01__00_depth_HIDDEN':'1', + '_0_wctrept__19__04__01__01_2_INT_L':hp.quniform('_0_wctrept__19__04__01__01_2_INT_L', 2.0, 20.0,1.0), + }, + ]), + }, + { + 'targetclass':'weka.classifiers.trees.RandomForest', + '_0_wctrf__20__01_INT_I':hp.qloguniform('_0_wctrf__20__01_INT_I', math.log(2.0), math.log(256.0), 1.0), + '_0_wctrf__20__02_auto_param_61':hp.choice('_0_wctrf__20__02_auto_param_61', [ + { + '_0_wctrf__20__02__00__00_features_HIDDEN':'0', + '_0_wctrf__20__02__00__01_1_INT_K':hp.choice('_0_wctrf__20__02__00__01_1_INT_K', ['1', ]), + }, + { + '_0_wctrf__20__02__01__00_features_HIDDEN':'1', + '_0_wctrf__20__02__01__01_2_INT_K':hp.qloguniform('_0_wctrf__20__02__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0), + }, + ]), + '_0_wctrf__20__03_auto_param_64':hp.choice('_0_wctrf__20__03_auto_param_64', [ + { + '_0_wctrf__20__03__00__00_depth_HIDDEN':'0', + '_0_wctrf__20__03__00__01_1_INT_depth':hp.choice('_0_wctrf__20__03__00__01_1_INT_depth', ['1', ]), + }, + { + '_0_wctrf__20__03__01__00_depth_HIDDEN':'1', + '_0_wctrf__20__03__01__01_2_INT_depth':hp.quniform('_0_wctrf__20__03__01__01_2_INT_depth', 2.0, 20.0,1.0), + }, + ]), + }, + { + 'targetclass':'weka.classifiers.trees.RandomTree', + '_0_wctrt__21__01_INT_M':hp.qloguniform('_0_wctrt__21__01_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_0_wctrt__21__02_U':hp.choice('_0_wctrt__21__02_U', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wctrt__21__03_auto_param_68':hp.choice('_0_wctrt__21__03_auto_param_68', [ + { + '_0_wctrt__21__03__00__00_features_HIDDEN':'0', + '_0_wctrt__21__03__00__01_1_INT_K':hp.choice('_0_wctrt__21__03__00__01_1_INT_K', ['0', ]), + }, + { + '_0_wctrt__21__03__01__00_features_HIDDEN':'1', + '_0_wctrt__21__03__01__01_2_INT_K':hp.qloguniform('_0_wctrt__21__03__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0), + }, + ]), + '_0_wctrt__21__04_auto_param_71':hp.choice('_0_wctrt__21__04_auto_param_71', [ + { + '_0_wctrt__21__04__00__00_depth_HIDDEN':'0', + '_0_wctrt__21__04__00__01_1_INT_depth':hp.choice('_0_wctrt__21__04__00__01_1_INT_depth', ['0', ]), + }, + { + '_0_wctrt__21__04__01__00_depth_HIDDEN':'1', + '_0_wctrt__21__04__01__01_2_INT_depth':hp.quniform('_0_wctrt__21__04__01__01_2_INT_depth', 2.0, 20.0,1.0), + }, + ]), + '_0_wctrt__21__05_auto_param_74':hp.choice('_0_wctrt__21__05_auto_param_74', [ + { + '_0_wctrt__21__05__00__00_back_HIDDEN':'0', + '_0_wctrt__21__05__00__01_1_INT_N':hp.choice('_0_wctrt__21__05__00__01_1_INT_N', ['0', ]), + }, + { + '_0_wctrt__21__05__01__00_back_HIDDEN':'1', + '_0_wctrt__21__05__01__01_2_INT_N':hp.quniform('_0_wctrt__21__05__01__01_2_INT_N', 2.0, 5.0,1.0), + }, + ]), + }, + ]), + hp.choice('is_meta', [ + ( + hp.choice('meta_choice', [ + { + 'targetclass':'weka.classifiers.lazy.LWL', + '_0_wcllwl__00__01_K':hp.choice('_0_wcllwl__00__01_K', ['120', '10', '30', '60', '90', '-1', ]), + '_0_wcllwl__00__02_U':hp.choice('_0_wcllwl__00__02_U', ['4', '1', '2', '3', '0', ]), + '_0_wcllwl__00__03_A':hp.choice('_0_wcllwl__00__03_A', ['weka.core.neighboursearch.LinearNNSearch', ]), + }, + { + 'targetclass':'weka.classifiers.meta.AdaBoostM1', + '_0_wcmabm__01__01_INT_I':hp.qloguniform('_0_wcmabm__01__01_INT_I', math.log(2.0), math.log(128.0), 1.0), + '_0_wcmabm__01__02_Q':hp.choice('_0_wcmabm__01__02_Q', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcmabm__01__03_S':hp.choice('_0_wcmabm__01__03_S', ['1', ]), + '_0_wcmabm__01__04_auto_param_80':hp.choice('_0_wcmabm__01__04_auto_param_80', [ + { + '_0_wcmabm__01__04__00__00_p_HIDDEN':'0', + '_0_wcmabm__01__04__00__01_1_P':hp.choice('_0_wcmabm__01__04__00__01_1_P', ['100', ]), + }, + { + '_0_wcmabm__01__04__01__00_p_HIDDEN':'1', + '_0_wcmabm__01__04__01__01_2_INT_P':hp.quniform('_0_wcmabm__01__04__01__01_2_INT_P', 50.0, 100.0,1.0), + }, + ]), + }, + { + 'targetclass':'weka.classifiers.meta.AttributeSelectedClassifier', + '_0_wcmasc__02__01_S':hp.choice('_0_wcmasc__02__01_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]), + '_0_wcmasc__02__02_E':hp.choice('_0_wcmasc__02__02_E', ['weka.attributeSelection.GainRatioAttributeEval', 'weka.attributeSelection.WrapperSubsetEval', 'weka.attributeSelection.OneRAttributeEval', 'weka.attributeSelection.InfoGainAttributeEval', 'weka.attributeSelection.HoldOutSubsetEvaluator', 'weka.attributeSelection.CfsSubsetEval', ]), + }, + { + 'targetclass':'weka.classifiers.meta.Bagging', + '_0_wcmb__03__01_INT_P':hp.quniform('_0_wcmb__03__01_INT_P', 10.0, 200.0,1.0), + '_0_wcmb__03__02_INT_I':hp.qloguniform('_0_wcmb__03__02_INT_I', math.log(2.0), math.log(128.0), 1.0), + '_0_wcmb__03__03_S':hp.choice('_0_wcmb__03__03_S', ['1', ]), + '_0_wcmb__03__04_O':hp.choice('_0_wcmb__03__04_O', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + 'targetclass':'weka.classifiers.meta.ClassificationViaRegression', + }, + { + 'targetclass':'weka.classifiers.meta.LogitBoost', + '_0_wcmlb__05__01_INT_I':hp.qloguniform('_0_wcmlb__05__01_INT_I', math.log(2.0), math.log(128.0), 1.0), + '_0_wcmlb__05__02_INT_R':hp.quniform('_0_wcmlb__05__02_INT_R', 1.0, 5.0,1.0), + '_0_wcmlb__05__03_Q':hp.choice('_0_wcmlb__05__03_Q', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcmlb__05__04_L':hp.choice('_0_wcmlb__05__04_L', ['1e50', ]), + '_0_wcmlb__05__05_S':hp.choice('_0_wcmlb__05__05_S', ['1', ]), + '_0_wcmlb__05__06_auto_param_87':hp.choice('_0_wcmlb__05__06_auto_param_87', [ + { + '_0_wcmlb__05__06__00__00_h_HIDDEN':'0', + '_0_wcmlb__05__06__00__01_1_H':hp.choice('_0_wcmlb__05__06__00__01_1_H', ['1', ]), + }, + { + '_0_wcmlb__05__06__01__00_h_HIDDEN':'1', + '_0_wcmlb__05__06__01__01_2_H':hp.uniform('_0_wcmlb__05__06__01__01_2_H', 0.0, 1.0), + }, + ]), + '_0_wcmlb__05__07_auto_param_90':hp.choice('_0_wcmlb__05__07_auto_param_90', [ + { + '_0_wcmlb__05__07__00__00_f_HIDDEN':'0', + '_0_wcmlb__05__07__00__01_1_F':hp.choice('_0_wcmlb__05__07__00__01_1_F', ['0', ]), + }, + { + '_0_wcmlb__05__07__01__00_f_HIDDEN':'1', + '_0_wcmlb__05__07__01__01_2_INT_F':hp.quniform('_0_wcmlb__05__07__01__01_2_INT_F', 1.0, 5.0,1.0), + }, + ]), + '_0_wcmlb__05__08_auto_param_93':hp.choice('_0_wcmlb__05__08_auto_param_93', [ + { + '_0_wcmlb__05__08__00__00_p_HIDDEN':'0', + '_0_wcmlb__05__08__00__01_1_P':hp.choice('_0_wcmlb__05__08__00__01_1_P', ['100', ]), + }, + { + '_0_wcmlb__05__08__01__00_p_HIDDEN':'1', + '_0_wcmlb__05__08__01__01_2_INT_P':hp.quniform('_0_wcmlb__05__08__01__01_2_INT_P', 50.0, 100.0,1.0), + }, + ]), + }, + { + 'targetclass':'weka.classifiers.meta.MultiClassClassifier', + '_0_wcmmcc__06__01_M':hp.choice('_0_wcmmcc__06__01_M', ['3', '1', '2', '0', ]), + '_0_wcmmcc__06__02_R':hp.uniform('_0_wcmmcc__06__02_R', 0.5, 4.0), + '_0_wcmmcc__06__03_P':hp.choice('_0_wcmmcc__06__03_P', ['REMOVE_PREV', 'REMOVED', ]), + '_0_wcmmcc__06__04_S':hp.choice('_0_wcmmcc__06__04_S', ['1', ]), + }, + { + 'targetclass':'weka.classifiers.meta.RandomCommittee', + '_0_wcmrc__07__01_INT_I':hp.qloguniform('_0_wcmrc__07__01_INT_I', math.log(2.0), math.log(64.0), 1.0), + '_0_wcmrc__07__02_S':hp.choice('_0_wcmrc__07__02_S', ['1', ]), + }, + { + 'targetclass':'weka.classifiers.meta.RandomSubSpace', + '_0_wcmrss__08__01_INT_I':hp.qloguniform('_0_wcmrss__08__01_INT_I', math.log(2.0), math.log(64.0), 1.0), + '_0_wcmrss__08__02_P':hp.uniform('_0_wcmrss__08__02_P', 0.1, 1.0), + '_0_wcmrss__08__03_S':hp.choice('_0_wcmrss__08__03_S', ['1', ]), + }, + ]), + hp.choice('meta_base_choice', [ + { + '_1_wcbbn_W':'weka.classifiers.bayes.BayesNet', + '_1_wcbbn_W-':'REMOVED', + '_1_wcbbn_W_00__02_D':hp.choice('_1_wcbbn_W_00__02_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wcbbn_W_00__03_Q':hp.choice('_1_wcbbn_W_00__03_Q', ['weka.classifiers.bayes.net.search.local.TAN', 'weka.classifiers.bayes.net.search.local.HillClimber', 'weka.classifiers.bayes.net.search.local.LAGDHillClimber', 'weka.classifiers.bayes.net.search.local.SimulatedAnnealing', 'weka.classifiers.bayes.net.search.local.TabuSearch', 'weka.classifiers.bayes.net.search.local.K2', ]), + }, + { + '_1_wcbnb_W':'weka.classifiers.bayes.NaiveBayes', + '_1_wcbnb_W-':'REMOVED', + '_1_wcbnb_W_01__02_K':hp.choice('_1_wcbnb_W_01__02_K', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wcbnb_W_01__03_D':hp.choice('_1_wcbnb_W_01__03_D', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_wcbnbm_W':'weka.classifiers.bayes.NaiveBayesMultinomial', + '_1_wcbnbm_W-':'REMOVED', + }, + { + '_1_wcfl_W':'weka.classifiers.functions.Logistic', + '_1_wcfl_W-':'REMOVED', + '_1_wcfl_W_03__02_R':hp.loguniform('_1_wcfl_W_03__02_R', math.log(1.0E-12), math.log(10.0)), + }, + { + '_1_wcfmp_W':'weka.classifiers.functions.MultilayerPerceptron', + '_1_wcfmp_W-':'REMOVED', + '_1_wcfmp_W_04__02_L':hp.uniform('_1_wcfmp_W_04__02_L', 0.1, 1.0), + '_1_wcfmp_W_04__03_M':hp.uniform('_1_wcfmp_W_04__03_M', 0.1, 1.0), + '_1_wcfmp_W_04__04_B':hp.choice('_1_wcfmp_W_04__04_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wcfmp_W_04__05_H':hp.choice('_1_wcfmp_W_04__05_H', ['t', 'i', 'o', 'a', ]), + '_1_wcfmp_W_04__06_C':hp.choice('_1_wcfmp_W_04__06_C', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wcfmp_W_04__07_R':hp.choice('_1_wcfmp_W_04__07_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wcfmp_W_04__08_D':hp.choice('_1_wcfmp_W_04__08_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wcfmp_W_04__09_S':hp.choice('_1_wcfmp_W_04__09_S', ['1', ]), + }, + { + '_1_wcfsgd_W':'weka.classifiers.functions.SGD', + '_1_wcfsgd_W-':'REMOVED', + '_1_wcfsgd_W_05__02_F':hp.choice('_1_wcfsgd_W_05__02_F', ['2', '1', '0', ]), + '_1_wcfsgd_W_05__03_L':hp.loguniform('_1_wcfsgd_W_05__03_L', math.log(1.0E-5), math.log(0.1)), + '_1_wcfsgd_W_05__04_R':hp.loguniform('_1_wcfsgd_W_05__04_R', math.log(1.0E-12), math.log(10.0)), + '_1_wcfsgd_W_05__05_N':hp.choice('_1_wcfsgd_W_05__05_N', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wcfsgd_W_05__06_M':hp.choice('_1_wcfsgd_W_05__06_M', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_wcfsmo_W':'weka.classifiers.functions.SMO', + '_1_wcfsmo_W-':'REMOVED', + '_1_wcfsmo_W_06__02_0_C':hp.uniform('_1_wcfsmo_W_06__02_0_C', 0.5, 1.5), + '_1_wcfsmo_W_06__03_1_N':hp.choice('_1_wcfsmo_W_06__03_1_N', ['2', '1', '0', ]), + '_1_wcfsmo_W_06__04_2_M':hp.choice('_1_wcfsmo_W_06__04_2_M', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wcfsmo_W_06__05_5_QUOTE_END':hp.choice('_1_wcfsmo_W_06__05_5_QUOTE_END', ['REMOVED', ]), + '_1_wcfsmo_W_06__06_auto_param_106':hp.choice('_1_wcfsmo_W_06__06_auto_param_106', [ + { + '_1_wcfsmo_W_06__06__00__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.NormalizedPolyKernel', + '_1_wcfsmo_W_06__06__00__01_4_npoly_E':hp.uniform('_1_wcfsmo_W_06__06__00__01_4_npoly_E', 0.2, 5.0), + '_1_wcfsmo_W_06__06__00__02_4_npoly_L':hp.choice('_1_wcfsmo_W_06__06__00__02_4_npoly_L', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_wcfsmo_W_06__06__01__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.PolyKernel', + '_1_wcfsmo_W_06__06__01__01_4_poly_E':hp.uniform('_1_wcfsmo_W_06__06__01__01_4_poly_E', 0.2, 5.0), + '_1_wcfsmo_W_06__06__01__02_4_poly_L':hp.choice('_1_wcfsmo_W_06__06__01__02_4_poly_L', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_wcfsmo_W_06__06__02__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.Puk', + '_1_wcfsmo_W_06__06__02__01_4_puk_S':hp.uniform('_1_wcfsmo_W_06__06__02__01_4_puk_S', 0.1, 10.0), + '_1_wcfsmo_W_06__06__02__02_4_puk_O':hp.uniform('_1_wcfsmo_W_06__06__02__02_4_puk_O', 0.1, 1.0), + }, + { + '_1_wcfsmo_W_06__06__03__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.RBFKernel', + '_1_wcfsmo_W_06__06__03__01_4_rbf_G':hp.loguniform('_1_wcfsmo_W_06__06__03__01_4_rbf_G', math.log(1.0E-4), math.log(1.0)), + }, + ]), + }, + { + '_1_wcfsl_W':'weka.classifiers.functions.SimpleLogistic', + '_1_wcfsl_W-':'REMOVED', + '_1_wcfsl_W_07__02_S':hp.choice('_1_wcfsl_W_07__02_S', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wcfsl_W_07__03_A':hp.choice('_1_wcfsl_W_07__03_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wcfsl_W_07__04_auto_param_112':hp.choice('_1_wcfsl_W_07__04_auto_param_112', [ + { + '_1_wcfsl_W_07__04__00__00_W_HIDDEN':'0', + '_1_wcfsl_W_07__04__00__01_1_W':hp.choice('_1_wcfsl_W_07__04__00__01_1_W', ['0', ]), + }, + { + '_1_wcfsl_W_07__04__01__00_W_HIDDEN':'1', + '_1_wcfsl_W_07__04__01__01_2_W':hp.uniform('_1_wcfsl_W_07__04__01__01_2_W', 0.0, 1.0), + }, + ]), + }, + { + '_1_wcfvp_W':'weka.classifiers.functions.VotedPerceptron', + '_1_wcfvp_W-':'REMOVED', + '_1_wcfvp_W_08__02_INT_I':hp.quniform('_1_wcfvp_W_08__02_INT_I', 1.0, 10.0,1.0), + '_1_wcfvp_W_08__03_INT_M':hp.qloguniform('_1_wcfvp_W_08__03_INT_M', math.log(5000.0), math.log(50000.0), 1.0), + '_1_wcfvp_W_08__04_E':hp.uniform('_1_wcfvp_W_08__04_E', 0.2, 5.0), + }, + { + '_1_wclib_W':'weka.classifiers.lazy.IBk', + '_1_wclib_W-':'REMOVED', + '_1_wclib_W_09__02_E':hp.choice('_1_wclib_W_09__02_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wclib_W_09__03_INT_K':hp.qloguniform('_1_wclib_W_09__03_INT_K', math.log(1.0), math.log(64.0), 1.0), + '_1_wclib_W_09__04_X':hp.choice('_1_wclib_W_09__04_X', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wclib_W_09__05_F':hp.choice('_1_wclib_W_09__05_F', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wclib_W_09__06_I':hp.choice('_1_wclib_W_09__06_I', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_wclks_W':'weka.classifiers.lazy.KStar', + '_1_wclks_W-':'REMOVED', + '_1_wclks_W_10__02_INT_B':hp.quniform('_1_wclks_W_10__02_INT_B', 1.0, 100.0,1.0), + '_1_wclks_W_10__03_E':hp.choice('_1_wclks_W_10__03_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wclks_W_10__04_M':hp.choice('_1_wclks_W_10__04_M', ['n', 'd', 'm', 'a', ]), + }, + { + '_1_wcrdt_W':'weka.classifiers.rules.DecisionTable', + '_1_wcrdt_W-':'REMOVED', + '_1_wcrdt_W_11__02_E':hp.choice('_1_wcrdt_W_11__02_E', ['auc', 'rmse', 'mae', 'acc', ]), + '_1_wcrdt_W_11__03_I':hp.choice('_1_wcrdt_W_11__03_I', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wcrdt_W_11__04_S':hp.choice('_1_wcrdt_W_11__04_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]), + '_1_wcrdt_W_11__05_X':hp.choice('_1_wcrdt_W_11__05_X', ['4', '2', '3', '1', ]), + }, + { + '_1_wcrjr_W':'weka.classifiers.rules.JRip', + '_1_wcrjr_W-':'REMOVED', + '_1_wcrjr_W_12__02_N':hp.uniform('_1_wcrjr_W_12__02_N', 1.0, 5.0), + '_1_wcrjr_W_12__03_E':hp.choice('_1_wcrjr_W_12__03_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wcrjr_W_12__04_P':hp.choice('_1_wcrjr_W_12__04_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wcrjr_W_12__05_INT_O':hp.quniform('_1_wcrjr_W_12__05_INT_O', 1.0, 5.0,1.0), + }, + { + '_1_wcror_W':'weka.classifiers.rules.OneR', + '_1_wcror_W-':'REMOVED', + '_1_wcror_W_13__02_INT_B':hp.qloguniform('_1_wcror_W_13__02_INT_B', math.log(1.0), math.log(32.0), 1.0), + }, + { + '_1_wcrpart_W':'weka.classifiers.rules.PART', + '_1_wcrpart_W-':'REMOVED', + '_1_wcrpart_W_14__02_INT_N':hp.quniform('_1_wcrpart_W_14__02_INT_N', 2.0, 5.0,1.0), + '_1_wcrpart_W_14__03_INT_M':hp.qloguniform('_1_wcrpart_W_14__03_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_wcrpart_W_14__04_R':hp.choice('_1_wcrpart_W_14__04_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wcrpart_W_14__05_B':hp.choice('_1_wcrpart_W_14__05_B', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_wcrzr_W':'weka.classifiers.rules.ZeroR', + '_1_wcrzr_W-':'REMOVED', + }, + { + '_1_wctds_W':'weka.classifiers.trees.DecisionStump', + '_1_wctds_W-':'REMOVED', + }, + { + '_1_wctj_W':'weka.classifiers.trees.J48', + '_1_wctj_W-':'REMOVED', + '_1_wctj_W_17__02_O':hp.choice('_1_wctj_W_17__02_O', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wctj_W_17__03_U':hp.choice('_1_wctj_W_17__03_U', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wctj_W_17__04_B':hp.choice('_1_wctj_W_17__04_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wctj_W_17__05_J':hp.choice('_1_wctj_W_17__05_J', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wctj_W_17__06_A':hp.choice('_1_wctj_W_17__06_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wctj_W_17__07_S':hp.choice('_1_wctj_W_17__07_S', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wctj_W_17__08_INT_M':hp.qloguniform('_1_wctj_W_17__08_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_wctj_W_17__09_C':hp.uniform('_1_wctj_W_17__09_C', 0.0, 1.0), + }, + { + '_1_wctlmt_W':'weka.classifiers.trees.LMT', + '_1_wctlmt_W-':'REMOVED', + '_1_wctlmt_W_18__02_B':hp.choice('_1_wctlmt_W_18__02_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wctlmt_W_18__03_R':hp.choice('_1_wctlmt_W_18__03_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wctlmt_W_18__04_C':hp.choice('_1_wctlmt_W_18__04_C', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wctlmt_W_18__05_P':hp.choice('_1_wctlmt_W_18__05_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wctlmt_W_18__06_INT_M':hp.qloguniform('_1_wctlmt_W_18__06_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_wctlmt_W_18__07_A':hp.choice('_1_wctlmt_W_18__07_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wctlmt_W_18__08_auto_param_126':hp.choice('_1_wctlmt_W_18__08_auto_param_126', [ + { + '_1_wctlmt_W_18__08__00__00_W_HIDDEN':'0', + '_1_wctlmt_W_18__08__00__01_1_W':hp.choice('_1_wctlmt_W_18__08__00__01_1_W', ['0', ]), + }, + { + '_1_wctlmt_W_18__08__01__00_W_HIDDEN':'1', + '_1_wctlmt_W_18__08__01__01_2_W':hp.uniform('_1_wctlmt_W_18__08__01__01_2_W', 0.0, 1.0), + }, + ]), + }, + { + '_1_wctrept_W':'weka.classifiers.trees.REPTree', + '_1_wctrept_W-':'REMOVED', + '_1_wctrept_W_19__02_INT_M':hp.qloguniform('_1_wctrept_W_19__02_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_wctrept_W_19__03_V':hp.loguniform('_1_wctrept_W_19__03_V', math.log(1.0E-5), math.log(0.1)), + '_1_wctrept_W_19__04_P':hp.choice('_1_wctrept_W_19__04_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wctrept_W_19__05_auto_param_130':hp.choice('_1_wctrept_W_19__05_auto_param_130', [ + { + '_1_wctrept_W_19__05__00__00_depth_HIDDEN':'0', + '_1_wctrept_W_19__05__00__01_1_INT_L':hp.choice('_1_wctrept_W_19__05__00__01_1_INT_L', ['-1', ]), + }, + { + '_1_wctrept_W_19__05__01__00_depth_HIDDEN':'1', + '_1_wctrept_W_19__05__01__01_2_INT_L':hp.quniform('_1_wctrept_W_19__05__01__01_2_INT_L', 2.0, 20.0,1.0), + }, + ]), + }, + { + '_1_wctrf_W':'weka.classifiers.trees.RandomForest', + '_1_wctrf_W-':'REMOVED', + '_1_wctrf_W_20__02_INT_I':hp.qloguniform('_1_wctrf_W_20__02_INT_I', math.log(2.0), math.log(256.0), 1.0), + '_1_wctrf_W_20__03_auto_param_134':hp.choice('_1_wctrf_W_20__03_auto_param_134', [ + { + '_1_wctrf_W_20__03__00__00_features_HIDDEN':'0', + '_1_wctrf_W_20__03__00__01_1_INT_K':hp.choice('_1_wctrf_W_20__03__00__01_1_INT_K', ['1', ]), + }, + { + '_1_wctrf_W_20__03__01__00_features_HIDDEN':'1', + '_1_wctrf_W_20__03__01__01_2_INT_K':hp.qloguniform('_1_wctrf_W_20__03__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0), + }, + ]), + '_1_wctrf_W_20__04_auto_param_137':hp.choice('_1_wctrf_W_20__04_auto_param_137', [ + { + '_1_wctrf_W_20__04__00__00_depth_HIDDEN':'0', + '_1_wctrf_W_20__04__00__01_1_INT_depth':hp.choice('_1_wctrf_W_20__04__00__01_1_INT_depth', ['1', ]), + }, + { + '_1_wctrf_W_20__04__01__00_depth_HIDDEN':'1', + '_1_wctrf_W_20__04__01__01_2_INT_depth':hp.quniform('_1_wctrf_W_20__04__01__01_2_INT_depth', 2.0, 20.0,1.0), + }, + ]), + }, + { + '_1_wctrt_W':'weka.classifiers.trees.RandomTree', + '_1_wctrt_W-':'REMOVED', + '_1_wctrt_W_21__02_INT_M':hp.qloguniform('_1_wctrt_W_21__02_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_wctrt_W_21__03_U':hp.choice('_1_wctrt_W_21__03_U', ['REMOVE_PREV', 'REMOVED', ]), + '_1_wctrt_W_21__04_auto_param_141':hp.choice('_1_wctrt_W_21__04_auto_param_141', [ + { + '_1_wctrt_W_21__04__00__00_features_HIDDEN':'0', + '_1_wctrt_W_21__04__00__01_1_INT_K':hp.choice('_1_wctrt_W_21__04__00__01_1_INT_K', ['0', ]), + }, + { + '_1_wctrt_W_21__04__01__00_features_HIDDEN':'1', + '_1_wctrt_W_21__04__01__01_2_INT_K':hp.qloguniform('_1_wctrt_W_21__04__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0), + }, + ]), + '_1_wctrt_W_21__05_auto_param_144':hp.choice('_1_wctrt_W_21__05_auto_param_144', [ + { + '_1_wctrt_W_21__05__00__00_depth_HIDDEN':'0', + '_1_wctrt_W_21__05__00__01_1_INT_depth':hp.choice('_1_wctrt_W_21__05__00__01_1_INT_depth', ['0', ]), + }, + { + '_1_wctrt_W_21__05__01__00_depth_HIDDEN':'1', + '_1_wctrt_W_21__05__01__01_2_INT_depth':hp.quniform('_1_wctrt_W_21__05__01__01_2_INT_depth', 2.0, 20.0,1.0), + }, + ]), + '_1_wctrt_W_21__06_auto_param_147':hp.choice('_1_wctrt_W_21__06_auto_param_147', [ + { + '_1_wctrt_W_21__06__00__00_back_HIDDEN':'0', + '_1_wctrt_W_21__06__00__01_1_INT_N':hp.choice('_1_wctrt_W_21__06__00__01_1_INT_N', ['0', ]), + }, + { + '_1_wctrt_W_21__06__01__00_back_HIDDEN':'1', + '_1_wctrt_W_21__06__01__01_2_INT_N':hp.quniform('_1_wctrt_W_21__06__01__01_2_INT_N', 2.0, 5.0,1.0), + }, + ]), + }, + ]), + ), + ( + hp.choice('ensemble_choice', [ + { + 'targetclass':'weka.classifiers.meta.Stacking', + '_0_wcms__00__01_X':hp.choice('_0_wcms__00__01_X', ['10', ]), + '_0_wcms__00__02_S':hp.choice('_0_wcms__00__02_S', ['1', ]), + }, + { + 'targetclass':'weka.classifiers.meta.Vote', + '_0_wcmv__01__01_R':hp.choice('_0_wcmv__01__01_R', ['MAX', 'PROD', 'MAJ', 'MIN', 'AVG', ]), + '_0_wcmv__01__02_S':hp.choice('_0_wcmv__01__02_S', ['1', ]), + }, + ]), + ( + hp.choice('ensemble_base_choice', [ + { + '_1_00_wcbbn_0_QUOTE_START_B':'weka.classifiers.bayes.BayesNet', + '_1_00_wcbbn_1__00__01_D':hp.choice('_1_00_wcbbn_1__00__01_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcbbn_1__00__02_Q':hp.choice('_1_00_wcbbn_1__00__02_Q', ['weka.classifiers.bayes.net.search.local.TAN', 'weka.classifiers.bayes.net.search.local.HillClimber', 'weka.classifiers.bayes.net.search.local.LAGDHillClimber', 'weka.classifiers.bayes.net.search.local.SimulatedAnnealing', 'weka.classifiers.bayes.net.search.local.TabuSearch', 'weka.classifiers.bayes.net.search.local.K2', ]), + '_1_00_wcbbn_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wcbnb_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayes', + '_1_00_wcbnb_1__01__01_K':hp.choice('_1_00_wcbnb_1__01__01_K', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcbnb_1__01__02_D':hp.choice('_1_00_wcbnb_1__01__02_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcbnb_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wcbnbm_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayesMultinomial', + '_1_00_wcbnbm_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wcfl_0_QUOTE_START_B':'weka.classifiers.functions.Logistic', + '_1_00_wcfl_1__03__01_R':hp.loguniform('_1_00_wcfl_1__03__01_R', math.log(1.0E-12), math.log(10.0)), + '_1_00_wcfl_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wcfmp_0_QUOTE_START_B':'weka.classifiers.functions.MultilayerPerceptron', + '_1_00_wcfmp_1__04__01_L':hp.uniform('_1_00_wcfmp_1__04__01_L', 0.1, 1.0), + '_1_00_wcfmp_1__04__02_M':hp.uniform('_1_00_wcfmp_1__04__02_M', 0.1, 1.0), + '_1_00_wcfmp_1__04__03_B':hp.choice('_1_00_wcfmp_1__04__03_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcfmp_1__04__04_H':hp.choice('_1_00_wcfmp_1__04__04_H', ['t', 'i', 'o', 'a', ]), + '_1_00_wcfmp_1__04__05_C':hp.choice('_1_00_wcfmp_1__04__05_C', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcfmp_1__04__06_R':hp.choice('_1_00_wcfmp_1__04__06_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcfmp_1__04__07_D':hp.choice('_1_00_wcfmp_1__04__07_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcfmp_1__04__08_S':hp.choice('_1_00_wcfmp_1__04__08_S', ['1', ]), + '_1_00_wcfmp_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wcfsgd_0_QUOTE_START_B':'weka.classifiers.functions.SGD', + '_1_00_wcfsgd_1__05__01_F':hp.choice('_1_00_wcfsgd_1__05__01_F', ['2', '1', '0', ]), + '_1_00_wcfsgd_1__05__02_L':hp.loguniform('_1_00_wcfsgd_1__05__02_L', math.log(1.0E-5), math.log(0.1)), + '_1_00_wcfsgd_1__05__03_R':hp.loguniform('_1_00_wcfsgd_1__05__03_R', math.log(1.0E-12), math.log(10.0)), + '_1_00_wcfsgd_1__05__04_N':hp.choice('_1_00_wcfsgd_1__05__04_N', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcfsgd_1__05__05_M':hp.choice('_1_00_wcfsgd_1__05__05_M', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcfsgd_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wcfsmo_0_QUOTE_START_B':'weka.classifiers.functions.SMO', + '_1_00_wcfsmo_1__06__01_0_C':hp.uniform('_1_00_wcfsmo_1__06__01_0_C', 0.5, 1.5), + '_1_00_wcfsmo_1__06__02_1_N':hp.choice('_1_00_wcfsmo_1__06__02_1_N', ['2', '1', '0', ]), + '_1_00_wcfsmo_1__06__03_2_M':hp.choice('_1_00_wcfsmo_1__06__03_2_M', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcfsmo_1__06__04_5_QUOTE_END':hp.choice('_1_00_wcfsmo_1__06__04_5_QUOTE_END', ['REMOVED', ]), + '_1_00_wcfsmo_1__06__05_auto_param_161':hp.choice('_1_00_wcfsmo_1__06__05_auto_param_161', [ + { + '_1_00_wcfsmo_1__06__05__00__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.NormalizedPolyKernel', + '_1_00_wcfsmo_1__06__05__00__01_4_npoly_E':hp.uniform('_1_00_wcfsmo_1__06__05__00__01_4_npoly_E', 0.2, 5.0), + '_1_00_wcfsmo_1__06__05__00__02_4_npoly_L':hp.choice('_1_00_wcfsmo_1__06__05__00__02_4_npoly_L', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_00_wcfsmo_1__06__05__01__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.PolyKernel', + '_1_00_wcfsmo_1__06__05__01__01_4_poly_E':hp.uniform('_1_00_wcfsmo_1__06__05__01__01_4_poly_E', 0.2, 5.0), + '_1_00_wcfsmo_1__06__05__01__02_4_poly_L':hp.choice('_1_00_wcfsmo_1__06__05__01__02_4_poly_L', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_00_wcfsmo_1__06__05__02__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.Puk', + '_1_00_wcfsmo_1__06__05__02__01_4_puk_S':hp.uniform('_1_00_wcfsmo_1__06__05__02__01_4_puk_S', 0.1, 10.0), + '_1_00_wcfsmo_1__06__05__02__02_4_puk_O':hp.uniform('_1_00_wcfsmo_1__06__05__02__02_4_puk_O', 0.1, 1.0), + }, + { + '_1_00_wcfsmo_1__06__05__03__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.RBFKernel', + '_1_00_wcfsmo_1__06__05__03__01_4_rbf_G':hp.loguniform('_1_00_wcfsmo_1__06__05__03__01_4_rbf_G', math.log(1.0E-4), math.log(1.0)), + }, + ]), + '_1_00_wcfsmo_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wcfsl_0_QUOTE_START_B':'weka.classifiers.functions.SimpleLogistic', + '_1_00_wcfsl_1__07__01_S':hp.choice('_1_00_wcfsl_1__07__01_S', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcfsl_1__07__02_A':hp.choice('_1_00_wcfsl_1__07__02_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcfsl_1__07__03_auto_param_167':hp.choice('_1_00_wcfsl_1__07__03_auto_param_167', [ + { + '_1_00_wcfsl_1__07__03__00__00_W_HIDDEN':'0', + '_1_00_wcfsl_1__07__03__00__01_1_W':hp.choice('_1_00_wcfsl_1__07__03__00__01_1_W', ['0', ]), + }, + { + '_1_00_wcfsl_1__07__03__01__00_W_HIDDEN':'1', + '_1_00_wcfsl_1__07__03__01__01_2_W':hp.uniform('_1_00_wcfsl_1__07__03__01__01_2_W', 0.0, 1.0), + }, + ]), + '_1_00_wcfsl_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wcfvp_0_QUOTE_START_B':'weka.classifiers.functions.VotedPerceptron', + '_1_00_wcfvp_1__08__01_INT_I':hp.quniform('_1_00_wcfvp_1__08__01_INT_I', 1.0, 10.0,1.0), + '_1_00_wcfvp_1__08__02_INT_M':hp.qloguniform('_1_00_wcfvp_1__08__02_INT_M', math.log(5000.0), math.log(50000.0), 1.0), + '_1_00_wcfvp_1__08__03_E':hp.uniform('_1_00_wcfvp_1__08__03_E', 0.2, 5.0), + '_1_00_wcfvp_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wclib_0_QUOTE_START_B':'weka.classifiers.lazy.IBk', + '_1_00_wclib_1__09__01_E':hp.choice('_1_00_wclib_1__09__01_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wclib_1__09__02_INT_K':hp.qloguniform('_1_00_wclib_1__09__02_INT_K', math.log(1.0), math.log(64.0), 1.0), + '_1_00_wclib_1__09__03_X':hp.choice('_1_00_wclib_1__09__03_X', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wclib_1__09__04_F':hp.choice('_1_00_wclib_1__09__04_F', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wclib_1__09__05_I':hp.choice('_1_00_wclib_1__09__05_I', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wclib_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wclks_0_QUOTE_START_B':'weka.classifiers.lazy.KStar', + '_1_00_wclks_1__10__01_INT_B':hp.quniform('_1_00_wclks_1__10__01_INT_B', 1.0, 100.0,1.0), + '_1_00_wclks_1__10__02_E':hp.choice('_1_00_wclks_1__10__02_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wclks_1__10__03_M':hp.choice('_1_00_wclks_1__10__03_M', ['n', 'd', 'm', 'a', ]), + '_1_00_wclks_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wcrdt_0_QUOTE_START_B':'weka.classifiers.rules.DecisionTable', + '_1_00_wcrdt_1__11__01_E':hp.choice('_1_00_wcrdt_1__11__01_E', ['auc', 'rmse', 'mae', 'acc', ]), + '_1_00_wcrdt_1__11__02_I':hp.choice('_1_00_wcrdt_1__11__02_I', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcrdt_1__11__03_S':hp.choice('_1_00_wcrdt_1__11__03_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]), + '_1_00_wcrdt_1__11__04_X':hp.choice('_1_00_wcrdt_1__11__04_X', ['4', '2', '3', '1', ]), + '_1_00_wcrdt_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wcrjr_0_QUOTE_START_B':'weka.classifiers.rules.JRip', + '_1_00_wcrjr_1__12__01_N':hp.uniform('_1_00_wcrjr_1__12__01_N', 1.0, 5.0), + '_1_00_wcrjr_1__12__02_E':hp.choice('_1_00_wcrjr_1__12__02_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcrjr_1__12__03_P':hp.choice('_1_00_wcrjr_1__12__03_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcrjr_1__12__04_INT_O':hp.quniform('_1_00_wcrjr_1__12__04_INT_O', 1.0, 5.0,1.0), + '_1_00_wcrjr_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wcror_0_QUOTE_START_B':'weka.classifiers.rules.OneR', + '_1_00_wcror_1__13__01_INT_B':hp.qloguniform('_1_00_wcror_1__13__01_INT_B', math.log(1.0), math.log(32.0), 1.0), + '_1_00_wcror_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wcrpart_0_QUOTE_START_B':'weka.classifiers.rules.PART', + '_1_00_wcrpart_1__14__01_INT_N':hp.quniform('_1_00_wcrpart_1__14__01_INT_N', 2.0, 5.0,1.0), + '_1_00_wcrpart_1__14__02_INT_M':hp.qloguniform('_1_00_wcrpart_1__14__02_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_00_wcrpart_1__14__03_R':hp.choice('_1_00_wcrpart_1__14__03_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcrpart_1__14__04_B':hp.choice('_1_00_wcrpart_1__14__04_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wcrpart_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wcrzr_0_QUOTE_START_B':'weka.classifiers.rules.ZeroR', + '_1_00_wcrzr_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wctds_0_QUOTE_START_B':'weka.classifiers.trees.DecisionStump', + '_1_00_wctds_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wctj_0_QUOTE_START_B':'weka.classifiers.trees.J48', + '_1_00_wctj_1__17__01_O':hp.choice('_1_00_wctj_1__17__01_O', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wctj_1__17__02_U':hp.choice('_1_00_wctj_1__17__02_U', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wctj_1__17__03_B':hp.choice('_1_00_wctj_1__17__03_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wctj_1__17__04_J':hp.choice('_1_00_wctj_1__17__04_J', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wctj_1__17__05_A':hp.choice('_1_00_wctj_1__17__05_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wctj_1__17__06_S':hp.choice('_1_00_wctj_1__17__06_S', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wctj_1__17__07_INT_M':hp.qloguniform('_1_00_wctj_1__17__07_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_00_wctj_1__17__08_C':hp.uniform('_1_00_wctj_1__17__08_C', 0.0, 1.0), + '_1_00_wctj_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wctlmt_0_QUOTE_START_B':'weka.classifiers.trees.LMT', + '_1_00_wctlmt_1__18__01_B':hp.choice('_1_00_wctlmt_1__18__01_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wctlmt_1__18__02_R':hp.choice('_1_00_wctlmt_1__18__02_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wctlmt_1__18__03_C':hp.choice('_1_00_wctlmt_1__18__03_C', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wctlmt_1__18__04_P':hp.choice('_1_00_wctlmt_1__18__04_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wctlmt_1__18__05_INT_M':hp.qloguniform('_1_00_wctlmt_1__18__05_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_00_wctlmt_1__18__06_A':hp.choice('_1_00_wctlmt_1__18__06_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wctlmt_1__18__07_auto_param_181':hp.choice('_1_00_wctlmt_1__18__07_auto_param_181', [ + { + '_1_00_wctlmt_1__18__07__00__00_W_HIDDEN':'0', + '_1_00_wctlmt_1__18__07__00__01_1_W':hp.choice('_1_00_wctlmt_1__18__07__00__01_1_W', ['0', ]), + }, + { + '_1_00_wctlmt_1__18__07__01__00_W_HIDDEN':'1', + '_1_00_wctlmt_1__18__07__01__01_2_W':hp.uniform('_1_00_wctlmt_1__18__07__01__01_2_W', 0.0, 1.0), + }, + ]), + '_1_00_wctlmt_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wctrept_0_QUOTE_START_B':'weka.classifiers.trees.REPTree', + '_1_00_wctrept_1__19__01_INT_M':hp.qloguniform('_1_00_wctrept_1__19__01_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_00_wctrept_1__19__02_V':hp.loguniform('_1_00_wctrept_1__19__02_V', math.log(1.0E-5), math.log(0.1)), + '_1_00_wctrept_1__19__03_P':hp.choice('_1_00_wctrept_1__19__03_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wctrept_1__19__04_auto_param_185':hp.choice('_1_00_wctrept_1__19__04_auto_param_185', [ + { + '_1_00_wctrept_1__19__04__00__00_depth_HIDDEN':'0', + '_1_00_wctrept_1__19__04__00__01_1_INT_L':hp.choice('_1_00_wctrept_1__19__04__00__01_1_INT_L', ['-1', ]), + }, + { + '_1_00_wctrept_1__19__04__01__00_depth_HIDDEN':'1', + '_1_00_wctrept_1__19__04__01__01_2_INT_L':hp.quniform('_1_00_wctrept_1__19__04__01__01_2_INT_L', 2.0, 20.0,1.0), + }, + ]), + '_1_00_wctrept_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wctrf_0_QUOTE_START_B':'weka.classifiers.trees.RandomForest', + '_1_00_wctrf_1__20__01_INT_I':hp.qloguniform('_1_00_wctrf_1__20__01_INT_I', math.log(2.0), math.log(256.0), 1.0), + '_1_00_wctrf_1__20__02_auto_param_189':hp.choice('_1_00_wctrf_1__20__02_auto_param_189', [ + { + '_1_00_wctrf_1__20__02__00__00_features_HIDDEN':'0', + '_1_00_wctrf_1__20__02__00__01_1_INT_K':hp.choice('_1_00_wctrf_1__20__02__00__01_1_INT_K', ['1', ]), + }, + { + '_1_00_wctrf_1__20__02__01__00_features_HIDDEN':'1', + '_1_00_wctrf_1__20__02__01__01_2_INT_K':hp.qloguniform('_1_00_wctrf_1__20__02__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0), + }, + ]), + '_1_00_wctrf_1__20__03_auto_param_192':hp.choice('_1_00_wctrf_1__20__03_auto_param_192', [ + { + '_1_00_wctrf_1__20__03__00__00_depth_HIDDEN':'0', + '_1_00_wctrf_1__20__03__00__01_1_INT_depth':hp.choice('_1_00_wctrf_1__20__03__00__01_1_INT_depth', ['1', ]), + }, + { + '_1_00_wctrf_1__20__03__01__00_depth_HIDDEN':'1', + '_1_00_wctrf_1__20__03__01__01_2_INT_depth':hp.quniform('_1_00_wctrf_1__20__03__01__01_2_INT_depth', 2.0, 20.0,1.0), + }, + ]), + '_1_00_wctrf_2_QUOTE_END':'REMOVED', + }, + { + '_1_00_wctrt_0_QUOTE_START_B':'weka.classifiers.trees.RandomTree', + '_1_00_wctrt_1__21__01_INT_M':hp.qloguniform('_1_00_wctrt_1__21__01_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_00_wctrt_1__21__02_U':hp.choice('_1_00_wctrt_1__21__02_U', ['REMOVE_PREV', 'REMOVED', ]), + '_1_00_wctrt_1__21__03_auto_param_196':hp.choice('_1_00_wctrt_1__21__03_auto_param_196', [ + { + '_1_00_wctrt_1__21__03__00__00_features_HIDDEN':'0', + '_1_00_wctrt_1__21__03__00__01_1_INT_K':hp.choice('_1_00_wctrt_1__21__03__00__01_1_INT_K', ['0', ]), + }, + { + '_1_00_wctrt_1__21__03__01__00_features_HIDDEN':'1', + '_1_00_wctrt_1__21__03__01__01_2_INT_K':hp.qloguniform('_1_00_wctrt_1__21__03__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0), + }, + ]), + '_1_00_wctrt_1__21__04_auto_param_199':hp.choice('_1_00_wctrt_1__21__04_auto_param_199', [ + { + '_1_00_wctrt_1__21__04__00__00_depth_HIDDEN':'0', + '_1_00_wctrt_1__21__04__00__01_1_INT_depth':hp.choice('_1_00_wctrt_1__21__04__00__01_1_INT_depth', ['0', ]), + }, + { + '_1_00_wctrt_1__21__04__01__00_depth_HIDDEN':'1', + '_1_00_wctrt_1__21__04__01__01_2_INT_depth':hp.quniform('_1_00_wctrt_1__21__04__01__01_2_INT_depth', 2.0, 20.0,1.0), + }, + ]), + '_1_00_wctrt_1__21__05_auto_param_202':hp.choice('_1_00_wctrt_1__21__05_auto_param_202', [ + { + '_1_00_wctrt_1__21__05__00__00_back_HIDDEN':'0', + '_1_00_wctrt_1__21__05__00__01_1_INT_N':hp.choice('_1_00_wctrt_1__21__05__00__01_1_INT_N', ['0', ]), + }, + { + '_1_00_wctrt_1__21__05__01__00_back_HIDDEN':'1', + '_1_00_wctrt_1__21__05__01__01_2_INT_N':hp.quniform('_1_00_wctrt_1__21__05__01__01_2_INT_N', 2.0, 5.0,1.0), + }, + ]), + '_1_00_wctrt_2_QUOTE_END':'REMOVED', + }, + ]), + hp.choice('auto_param_205', [ + ( + ), + ( + hp.choice('auto_param_208', [ + { + '_1_01_wcbbn_0_QUOTE_START_B':'weka.classifiers.bayes.BayesNet', + '_1_01_wcbbn_1__00__01_D':hp.choice('_1_01_wcbbn_1__00__01_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcbbn_1__00__02_Q':hp.choice('_1_01_wcbbn_1__00__02_Q', ['weka.classifiers.bayes.net.search.local.TAN', 'weka.classifiers.bayes.net.search.local.HillClimber', 'weka.classifiers.bayes.net.search.local.LAGDHillClimber', 'weka.classifiers.bayes.net.search.local.SimulatedAnnealing', 'weka.classifiers.bayes.net.search.local.TabuSearch', 'weka.classifiers.bayes.net.search.local.K2', ]), + '_1_01_wcbbn_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wcbnb_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayes', + '_1_01_wcbnb_1__01__01_K':hp.choice('_1_01_wcbnb_1__01__01_K', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcbnb_1__01__02_D':hp.choice('_1_01_wcbnb_1__01__02_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcbnb_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wcbnbm_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayesMultinomial', + '_1_01_wcbnbm_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wcfl_0_QUOTE_START_B':'weka.classifiers.functions.Logistic', + '_1_01_wcfl_1__03__01_R':hp.loguniform('_1_01_wcfl_1__03__01_R', math.log(1.0E-12), math.log(10.0)), + '_1_01_wcfl_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wcfmp_0_QUOTE_START_B':'weka.classifiers.functions.MultilayerPerceptron', + '_1_01_wcfmp_1__04__01_L':hp.uniform('_1_01_wcfmp_1__04__01_L', 0.1, 1.0), + '_1_01_wcfmp_1__04__02_M':hp.uniform('_1_01_wcfmp_1__04__02_M', 0.1, 1.0), + '_1_01_wcfmp_1__04__03_B':hp.choice('_1_01_wcfmp_1__04__03_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcfmp_1__04__04_H':hp.choice('_1_01_wcfmp_1__04__04_H', ['t', 'i', 'o', 'a', ]), + '_1_01_wcfmp_1__04__05_C':hp.choice('_1_01_wcfmp_1__04__05_C', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcfmp_1__04__06_R':hp.choice('_1_01_wcfmp_1__04__06_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcfmp_1__04__07_D':hp.choice('_1_01_wcfmp_1__04__07_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcfmp_1__04__08_S':hp.choice('_1_01_wcfmp_1__04__08_S', ['1', ]), + '_1_01_wcfmp_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wcfsgd_0_QUOTE_START_B':'weka.classifiers.functions.SGD', + '_1_01_wcfsgd_1__05__01_F':hp.choice('_1_01_wcfsgd_1__05__01_F', ['2', '1', '0', ]), + '_1_01_wcfsgd_1__05__02_L':hp.loguniform('_1_01_wcfsgd_1__05__02_L', math.log(1.0E-5), math.log(0.1)), + '_1_01_wcfsgd_1__05__03_R':hp.loguniform('_1_01_wcfsgd_1__05__03_R', math.log(1.0E-12), math.log(10.0)), + '_1_01_wcfsgd_1__05__04_N':hp.choice('_1_01_wcfsgd_1__05__04_N', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcfsgd_1__05__05_M':hp.choice('_1_01_wcfsgd_1__05__05_M', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcfsgd_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wcfsmo_0_QUOTE_START_B':'weka.classifiers.functions.SMO', + '_1_01_wcfsmo_1__06__01_0_C':hp.uniform('_1_01_wcfsmo_1__06__01_0_C', 0.5, 1.5), + '_1_01_wcfsmo_1__06__02_1_N':hp.choice('_1_01_wcfsmo_1__06__02_1_N', ['2', '1', '0', ]), + '_1_01_wcfsmo_1__06__03_2_M':hp.choice('_1_01_wcfsmo_1__06__03_2_M', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcfsmo_1__06__04_5_QUOTE_END':hp.choice('_1_01_wcfsmo_1__06__04_5_QUOTE_END', ['REMOVED', ]), + '_1_01_wcfsmo_1__06__05_auto_param_216':hp.choice('_1_01_wcfsmo_1__06__05_auto_param_216', [ + { + '_1_01_wcfsmo_1__06__05__00__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.NormalizedPolyKernel', + '_1_01_wcfsmo_1__06__05__00__01_4_npoly_E':hp.uniform('_1_01_wcfsmo_1__06__05__00__01_4_npoly_E', 0.2, 5.0), + '_1_01_wcfsmo_1__06__05__00__02_4_npoly_L':hp.choice('_1_01_wcfsmo_1__06__05__00__02_4_npoly_L', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_01_wcfsmo_1__06__05__01__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.PolyKernel', + '_1_01_wcfsmo_1__06__05__01__01_4_poly_E':hp.uniform('_1_01_wcfsmo_1__06__05__01__01_4_poly_E', 0.2, 5.0), + '_1_01_wcfsmo_1__06__05__01__02_4_poly_L':hp.choice('_1_01_wcfsmo_1__06__05__01__02_4_poly_L', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_01_wcfsmo_1__06__05__02__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.Puk', + '_1_01_wcfsmo_1__06__05__02__01_4_puk_S':hp.uniform('_1_01_wcfsmo_1__06__05__02__01_4_puk_S', 0.1, 10.0), + '_1_01_wcfsmo_1__06__05__02__02_4_puk_O':hp.uniform('_1_01_wcfsmo_1__06__05__02__02_4_puk_O', 0.1, 1.0), + }, + { + '_1_01_wcfsmo_1__06__05__03__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.RBFKernel', + '_1_01_wcfsmo_1__06__05__03__01_4_rbf_G':hp.loguniform('_1_01_wcfsmo_1__06__05__03__01_4_rbf_G', math.log(1.0E-4), math.log(1.0)), + }, + ]), + '_1_01_wcfsmo_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wcfsl_0_QUOTE_START_B':'weka.classifiers.functions.SimpleLogistic', + '_1_01_wcfsl_1__07__01_S':hp.choice('_1_01_wcfsl_1__07__01_S', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcfsl_1__07__02_A':hp.choice('_1_01_wcfsl_1__07__02_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcfsl_1__07__03_auto_param_222':hp.choice('_1_01_wcfsl_1__07__03_auto_param_222', [ + { + '_1_01_wcfsl_1__07__03__00__00_W_HIDDEN':'0', + '_1_01_wcfsl_1__07__03__00__01_1_W':hp.choice('_1_01_wcfsl_1__07__03__00__01_1_W', ['0', ]), + }, + { + '_1_01_wcfsl_1__07__03__01__00_W_HIDDEN':'1', + '_1_01_wcfsl_1__07__03__01__01_2_W':hp.uniform('_1_01_wcfsl_1__07__03__01__01_2_W', 0.0, 1.0), + }, + ]), + '_1_01_wcfsl_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wcfvp_0_QUOTE_START_B':'weka.classifiers.functions.VotedPerceptron', + '_1_01_wcfvp_1__08__01_INT_I':hp.quniform('_1_01_wcfvp_1__08__01_INT_I', 1.0, 10.0,1.0), + '_1_01_wcfvp_1__08__02_INT_M':hp.qloguniform('_1_01_wcfvp_1__08__02_INT_M', math.log(5000.0), math.log(50000.0), 1.0), + '_1_01_wcfvp_1__08__03_E':hp.uniform('_1_01_wcfvp_1__08__03_E', 0.2, 5.0), + '_1_01_wcfvp_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wclib_0_QUOTE_START_B':'weka.classifiers.lazy.IBk', + '_1_01_wclib_1__09__01_E':hp.choice('_1_01_wclib_1__09__01_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wclib_1__09__02_INT_K':hp.qloguniform('_1_01_wclib_1__09__02_INT_K', math.log(1.0), math.log(64.0), 1.0), + '_1_01_wclib_1__09__03_X':hp.choice('_1_01_wclib_1__09__03_X', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wclib_1__09__04_F':hp.choice('_1_01_wclib_1__09__04_F', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wclib_1__09__05_I':hp.choice('_1_01_wclib_1__09__05_I', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wclib_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wclks_0_QUOTE_START_B':'weka.classifiers.lazy.KStar', + '_1_01_wclks_1__10__01_INT_B':hp.quniform('_1_01_wclks_1__10__01_INT_B', 1.0, 100.0,1.0), + '_1_01_wclks_1__10__02_E':hp.choice('_1_01_wclks_1__10__02_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wclks_1__10__03_M':hp.choice('_1_01_wclks_1__10__03_M', ['n', 'd', 'm', 'a', ]), + '_1_01_wclks_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wcrdt_0_QUOTE_START_B':'weka.classifiers.rules.DecisionTable', + '_1_01_wcrdt_1__11__01_E':hp.choice('_1_01_wcrdt_1__11__01_E', ['auc', 'rmse', 'mae', 'acc', ]), + '_1_01_wcrdt_1__11__02_I':hp.choice('_1_01_wcrdt_1__11__02_I', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcrdt_1__11__03_S':hp.choice('_1_01_wcrdt_1__11__03_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]), + '_1_01_wcrdt_1__11__04_X':hp.choice('_1_01_wcrdt_1__11__04_X', ['4', '2', '3', '1', ]), + '_1_01_wcrdt_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wcrjr_0_QUOTE_START_B':'weka.classifiers.rules.JRip', + '_1_01_wcrjr_1__12__01_N':hp.uniform('_1_01_wcrjr_1__12__01_N', 1.0, 5.0), + '_1_01_wcrjr_1__12__02_E':hp.choice('_1_01_wcrjr_1__12__02_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcrjr_1__12__03_P':hp.choice('_1_01_wcrjr_1__12__03_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcrjr_1__12__04_INT_O':hp.quniform('_1_01_wcrjr_1__12__04_INT_O', 1.0, 5.0,1.0), + '_1_01_wcrjr_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wcror_0_QUOTE_START_B':'weka.classifiers.rules.OneR', + '_1_01_wcror_1__13__01_INT_B':hp.qloguniform('_1_01_wcror_1__13__01_INT_B', math.log(1.0), math.log(32.0), 1.0), + '_1_01_wcror_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wcrpart_0_QUOTE_START_B':'weka.classifiers.rules.PART', + '_1_01_wcrpart_1__14__01_INT_N':hp.quniform('_1_01_wcrpart_1__14__01_INT_N', 2.0, 5.0,1.0), + '_1_01_wcrpart_1__14__02_INT_M':hp.qloguniform('_1_01_wcrpart_1__14__02_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_01_wcrpart_1__14__03_R':hp.choice('_1_01_wcrpart_1__14__03_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcrpart_1__14__04_B':hp.choice('_1_01_wcrpart_1__14__04_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wcrpart_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wcrzr_0_QUOTE_START_B':'weka.classifiers.rules.ZeroR', + '_1_01_wcrzr_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wctds_0_QUOTE_START_B':'weka.classifiers.trees.DecisionStump', + '_1_01_wctds_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wctj_0_QUOTE_START_B':'weka.classifiers.trees.J48', + '_1_01_wctj_1__17__01_O':hp.choice('_1_01_wctj_1__17__01_O', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wctj_1__17__02_U':hp.choice('_1_01_wctj_1__17__02_U', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wctj_1__17__03_B':hp.choice('_1_01_wctj_1__17__03_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wctj_1__17__04_J':hp.choice('_1_01_wctj_1__17__04_J', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wctj_1__17__05_A':hp.choice('_1_01_wctj_1__17__05_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wctj_1__17__06_S':hp.choice('_1_01_wctj_1__17__06_S', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wctj_1__17__07_INT_M':hp.qloguniform('_1_01_wctj_1__17__07_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_01_wctj_1__17__08_C':hp.uniform('_1_01_wctj_1__17__08_C', 0.0, 1.0), + '_1_01_wctj_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wctlmt_0_QUOTE_START_B':'weka.classifiers.trees.LMT', + '_1_01_wctlmt_1__18__01_B':hp.choice('_1_01_wctlmt_1__18__01_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wctlmt_1__18__02_R':hp.choice('_1_01_wctlmt_1__18__02_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wctlmt_1__18__03_C':hp.choice('_1_01_wctlmt_1__18__03_C', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wctlmt_1__18__04_P':hp.choice('_1_01_wctlmt_1__18__04_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wctlmt_1__18__05_INT_M':hp.qloguniform('_1_01_wctlmt_1__18__05_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_01_wctlmt_1__18__06_A':hp.choice('_1_01_wctlmt_1__18__06_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wctlmt_1__18__07_auto_param_236':hp.choice('_1_01_wctlmt_1__18__07_auto_param_236', [ + { + '_1_01_wctlmt_1__18__07__00__00_W_HIDDEN':'0', + '_1_01_wctlmt_1__18__07__00__01_1_W':hp.choice('_1_01_wctlmt_1__18__07__00__01_1_W', ['0', ]), + }, + { + '_1_01_wctlmt_1__18__07__01__00_W_HIDDEN':'1', + '_1_01_wctlmt_1__18__07__01__01_2_W':hp.uniform('_1_01_wctlmt_1__18__07__01__01_2_W', 0.0, 1.0), + }, + ]), + '_1_01_wctlmt_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wctrept_0_QUOTE_START_B':'weka.classifiers.trees.REPTree', + '_1_01_wctrept_1__19__01_INT_M':hp.qloguniform('_1_01_wctrept_1__19__01_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_01_wctrept_1__19__02_V':hp.loguniform('_1_01_wctrept_1__19__02_V', math.log(1.0E-5), math.log(0.1)), + '_1_01_wctrept_1__19__03_P':hp.choice('_1_01_wctrept_1__19__03_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wctrept_1__19__04_auto_param_240':hp.choice('_1_01_wctrept_1__19__04_auto_param_240', [ + { + '_1_01_wctrept_1__19__04__00__00_depth_HIDDEN':'0', + '_1_01_wctrept_1__19__04__00__01_1_INT_L':hp.choice('_1_01_wctrept_1__19__04__00__01_1_INT_L', ['-1', ]), + }, + { + '_1_01_wctrept_1__19__04__01__00_depth_HIDDEN':'1', + '_1_01_wctrept_1__19__04__01__01_2_INT_L':hp.quniform('_1_01_wctrept_1__19__04__01__01_2_INT_L', 2.0, 20.0,1.0), + }, + ]), + '_1_01_wctrept_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wctrf_0_QUOTE_START_B':'weka.classifiers.trees.RandomForest', + '_1_01_wctrf_1__20__01_INT_I':hp.qloguniform('_1_01_wctrf_1__20__01_INT_I', math.log(2.0), math.log(256.0), 1.0), + '_1_01_wctrf_1__20__02_auto_param_244':hp.choice('_1_01_wctrf_1__20__02_auto_param_244', [ + { + '_1_01_wctrf_1__20__02__00__00_features_HIDDEN':'0', + '_1_01_wctrf_1__20__02__00__01_1_INT_K':hp.choice('_1_01_wctrf_1__20__02__00__01_1_INT_K', ['1', ]), + }, + { + '_1_01_wctrf_1__20__02__01__00_features_HIDDEN':'1', + '_1_01_wctrf_1__20__02__01__01_2_INT_K':hp.qloguniform('_1_01_wctrf_1__20__02__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0), + }, + ]), + '_1_01_wctrf_1__20__03_auto_param_247':hp.choice('_1_01_wctrf_1__20__03_auto_param_247', [ + { + '_1_01_wctrf_1__20__03__00__00_depth_HIDDEN':'0', + '_1_01_wctrf_1__20__03__00__01_1_INT_depth':hp.choice('_1_01_wctrf_1__20__03__00__01_1_INT_depth', ['1', ]), + }, + { + '_1_01_wctrf_1__20__03__01__00_depth_HIDDEN':'1', + '_1_01_wctrf_1__20__03__01__01_2_INT_depth':hp.quniform('_1_01_wctrf_1__20__03__01__01_2_INT_depth', 2.0, 20.0,1.0), + }, + ]), + '_1_01_wctrf_2_QUOTE_END':'REMOVED', + }, + { + '_1_01_wctrt_0_QUOTE_START_B':'weka.classifiers.trees.RandomTree', + '_1_01_wctrt_1__21__01_INT_M':hp.qloguniform('_1_01_wctrt_1__21__01_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_01_wctrt_1__21__02_U':hp.choice('_1_01_wctrt_1__21__02_U', ['REMOVE_PREV', 'REMOVED', ]), + '_1_01_wctrt_1__21__03_auto_param_251':hp.choice('_1_01_wctrt_1__21__03_auto_param_251', [ + { + '_1_01_wctrt_1__21__03__00__00_features_HIDDEN':'0', + '_1_01_wctrt_1__21__03__00__01_1_INT_K':hp.choice('_1_01_wctrt_1__21__03__00__01_1_INT_K', ['0', ]), + }, + { + '_1_01_wctrt_1__21__03__01__00_features_HIDDEN':'1', + '_1_01_wctrt_1__21__03__01__01_2_INT_K':hp.qloguniform('_1_01_wctrt_1__21__03__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0), + }, + ]), + '_1_01_wctrt_1__21__04_auto_param_254':hp.choice('_1_01_wctrt_1__21__04_auto_param_254', [ + { + '_1_01_wctrt_1__21__04__00__00_depth_HIDDEN':'0', + '_1_01_wctrt_1__21__04__00__01_1_INT_depth':hp.choice('_1_01_wctrt_1__21__04__00__01_1_INT_depth', ['0', ]), + }, + { + '_1_01_wctrt_1__21__04__01__00_depth_HIDDEN':'1', + '_1_01_wctrt_1__21__04__01__01_2_INT_depth':hp.quniform('_1_01_wctrt_1__21__04__01__01_2_INT_depth', 2.0, 20.0,1.0), + }, + ]), + '_1_01_wctrt_1__21__05_auto_param_257':hp.choice('_1_01_wctrt_1__21__05_auto_param_257', [ + { + '_1_01_wctrt_1__21__05__00__00_back_HIDDEN':'0', + '_1_01_wctrt_1__21__05__00__01_1_INT_N':hp.choice('_1_01_wctrt_1__21__05__00__01_1_INT_N', ['0', ]), + }, + { + '_1_01_wctrt_1__21__05__01__00_back_HIDDEN':'1', + '_1_01_wctrt_1__21__05__01__01_2_INT_N':hp.quniform('_1_01_wctrt_1__21__05__01__01_2_INT_N', 2.0, 5.0,1.0), + }, + ]), + '_1_01_wctrt_2_QUOTE_END':'REMOVED', + }, + ]), + hp.choice('auto_param_260', [ + ( + ), + ( + hp.choice('auto_param_263', [ + { + '_1_02_wcbbn_0_QUOTE_START_B':'weka.classifiers.bayes.BayesNet', + '_1_02_wcbbn_1__00__01_D':hp.choice('_1_02_wcbbn_1__00__01_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcbbn_1__00__02_Q':hp.choice('_1_02_wcbbn_1__00__02_Q', ['weka.classifiers.bayes.net.search.local.TAN', 'weka.classifiers.bayes.net.search.local.HillClimber', 'weka.classifiers.bayes.net.search.local.LAGDHillClimber', 'weka.classifiers.bayes.net.search.local.SimulatedAnnealing', 'weka.classifiers.bayes.net.search.local.TabuSearch', 'weka.classifiers.bayes.net.search.local.K2', ]), + '_1_02_wcbbn_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wcbnb_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayes', + '_1_02_wcbnb_1__01__01_K':hp.choice('_1_02_wcbnb_1__01__01_K', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcbnb_1__01__02_D':hp.choice('_1_02_wcbnb_1__01__02_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcbnb_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wcbnbm_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayesMultinomial', + '_1_02_wcbnbm_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wcfl_0_QUOTE_START_B':'weka.classifiers.functions.Logistic', + '_1_02_wcfl_1__03__01_R':hp.loguniform('_1_02_wcfl_1__03__01_R', math.log(1.0E-12), math.log(10.0)), + '_1_02_wcfl_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wcfmp_0_QUOTE_START_B':'weka.classifiers.functions.MultilayerPerceptron', + '_1_02_wcfmp_1__04__01_L':hp.uniform('_1_02_wcfmp_1__04__01_L', 0.1, 1.0), + '_1_02_wcfmp_1__04__02_M':hp.uniform('_1_02_wcfmp_1__04__02_M', 0.1, 1.0), + '_1_02_wcfmp_1__04__03_B':hp.choice('_1_02_wcfmp_1__04__03_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcfmp_1__04__04_H':hp.choice('_1_02_wcfmp_1__04__04_H', ['t', 'i', 'o', 'a', ]), + '_1_02_wcfmp_1__04__05_C':hp.choice('_1_02_wcfmp_1__04__05_C', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcfmp_1__04__06_R':hp.choice('_1_02_wcfmp_1__04__06_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcfmp_1__04__07_D':hp.choice('_1_02_wcfmp_1__04__07_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcfmp_1__04__08_S':hp.choice('_1_02_wcfmp_1__04__08_S', ['1', ]), + '_1_02_wcfmp_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wcfsgd_0_QUOTE_START_B':'weka.classifiers.functions.SGD', + '_1_02_wcfsgd_1__05__01_F':hp.choice('_1_02_wcfsgd_1__05__01_F', ['2', '1', '0', ]), + '_1_02_wcfsgd_1__05__02_L':hp.loguniform('_1_02_wcfsgd_1__05__02_L', math.log(1.0E-5), math.log(0.1)), + '_1_02_wcfsgd_1__05__03_R':hp.loguniform('_1_02_wcfsgd_1__05__03_R', math.log(1.0E-12), math.log(10.0)), + '_1_02_wcfsgd_1__05__04_N':hp.choice('_1_02_wcfsgd_1__05__04_N', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcfsgd_1__05__05_M':hp.choice('_1_02_wcfsgd_1__05__05_M', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcfsgd_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wcfsmo_0_QUOTE_START_B':'weka.classifiers.functions.SMO', + '_1_02_wcfsmo_1__06__01_0_C':hp.uniform('_1_02_wcfsmo_1__06__01_0_C', 0.5, 1.5), + '_1_02_wcfsmo_1__06__02_1_N':hp.choice('_1_02_wcfsmo_1__06__02_1_N', ['2', '1', '0', ]), + '_1_02_wcfsmo_1__06__03_2_M':hp.choice('_1_02_wcfsmo_1__06__03_2_M', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcfsmo_1__06__04_5_QUOTE_END':hp.choice('_1_02_wcfsmo_1__06__04_5_QUOTE_END', ['REMOVED', ]), + '_1_02_wcfsmo_1__06__05_auto_param_271':hp.choice('_1_02_wcfsmo_1__06__05_auto_param_271', [ + { + '_1_02_wcfsmo_1__06__05__00__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.NormalizedPolyKernel', + '_1_02_wcfsmo_1__06__05__00__01_4_npoly_E':hp.uniform('_1_02_wcfsmo_1__06__05__00__01_4_npoly_E', 0.2, 5.0), + '_1_02_wcfsmo_1__06__05__00__02_4_npoly_L':hp.choice('_1_02_wcfsmo_1__06__05__00__02_4_npoly_L', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_02_wcfsmo_1__06__05__01__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.PolyKernel', + '_1_02_wcfsmo_1__06__05__01__01_4_poly_E':hp.uniform('_1_02_wcfsmo_1__06__05__01__01_4_poly_E', 0.2, 5.0), + '_1_02_wcfsmo_1__06__05__01__02_4_poly_L':hp.choice('_1_02_wcfsmo_1__06__05__01__02_4_poly_L', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_02_wcfsmo_1__06__05__02__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.Puk', + '_1_02_wcfsmo_1__06__05__02__01_4_puk_S':hp.uniform('_1_02_wcfsmo_1__06__05__02__01_4_puk_S', 0.1, 10.0), + '_1_02_wcfsmo_1__06__05__02__02_4_puk_O':hp.uniform('_1_02_wcfsmo_1__06__05__02__02_4_puk_O', 0.1, 1.0), + }, + { + '_1_02_wcfsmo_1__06__05__03__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.RBFKernel', + '_1_02_wcfsmo_1__06__05__03__01_4_rbf_G':hp.loguniform('_1_02_wcfsmo_1__06__05__03__01_4_rbf_G', math.log(1.0E-4), math.log(1.0)), + }, + ]), + '_1_02_wcfsmo_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wcfsl_0_QUOTE_START_B':'weka.classifiers.functions.SimpleLogistic', + '_1_02_wcfsl_1__07__01_S':hp.choice('_1_02_wcfsl_1__07__01_S', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcfsl_1__07__02_A':hp.choice('_1_02_wcfsl_1__07__02_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcfsl_1__07__03_auto_param_277':hp.choice('_1_02_wcfsl_1__07__03_auto_param_277', [ + { + '_1_02_wcfsl_1__07__03__00__00_W_HIDDEN':'0', + '_1_02_wcfsl_1__07__03__00__01_1_W':hp.choice('_1_02_wcfsl_1__07__03__00__01_1_W', ['0', ]), + }, + { + '_1_02_wcfsl_1__07__03__01__00_W_HIDDEN':'1', + '_1_02_wcfsl_1__07__03__01__01_2_W':hp.uniform('_1_02_wcfsl_1__07__03__01__01_2_W', 0.0, 1.0), + }, + ]), + '_1_02_wcfsl_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wcfvp_0_QUOTE_START_B':'weka.classifiers.functions.VotedPerceptron', + '_1_02_wcfvp_1__08__01_INT_I':hp.quniform('_1_02_wcfvp_1__08__01_INT_I', 1.0, 10.0,1.0), + '_1_02_wcfvp_1__08__02_INT_M':hp.qloguniform('_1_02_wcfvp_1__08__02_INT_M', math.log(5000.0), math.log(50000.0), 1.0), + '_1_02_wcfvp_1__08__03_E':hp.uniform('_1_02_wcfvp_1__08__03_E', 0.2, 5.0), + '_1_02_wcfvp_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wclib_0_QUOTE_START_B':'weka.classifiers.lazy.IBk', + '_1_02_wclib_1__09__01_E':hp.choice('_1_02_wclib_1__09__01_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wclib_1__09__02_INT_K':hp.qloguniform('_1_02_wclib_1__09__02_INT_K', math.log(1.0), math.log(64.0), 1.0), + '_1_02_wclib_1__09__03_X':hp.choice('_1_02_wclib_1__09__03_X', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wclib_1__09__04_F':hp.choice('_1_02_wclib_1__09__04_F', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wclib_1__09__05_I':hp.choice('_1_02_wclib_1__09__05_I', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wclib_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wclks_0_QUOTE_START_B':'weka.classifiers.lazy.KStar', + '_1_02_wclks_1__10__01_INT_B':hp.quniform('_1_02_wclks_1__10__01_INT_B', 1.0, 100.0,1.0), + '_1_02_wclks_1__10__02_E':hp.choice('_1_02_wclks_1__10__02_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wclks_1__10__03_M':hp.choice('_1_02_wclks_1__10__03_M', ['n', 'd', 'm', 'a', ]), + '_1_02_wclks_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wcrdt_0_QUOTE_START_B':'weka.classifiers.rules.DecisionTable', + '_1_02_wcrdt_1__11__01_E':hp.choice('_1_02_wcrdt_1__11__01_E', ['auc', 'rmse', 'mae', 'acc', ]), + '_1_02_wcrdt_1__11__02_I':hp.choice('_1_02_wcrdt_1__11__02_I', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcrdt_1__11__03_S':hp.choice('_1_02_wcrdt_1__11__03_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]), + '_1_02_wcrdt_1__11__04_X':hp.choice('_1_02_wcrdt_1__11__04_X', ['4', '2', '3', '1', ]), + '_1_02_wcrdt_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wcrjr_0_QUOTE_START_B':'weka.classifiers.rules.JRip', + '_1_02_wcrjr_1__12__01_N':hp.uniform('_1_02_wcrjr_1__12__01_N', 1.0, 5.0), + '_1_02_wcrjr_1__12__02_E':hp.choice('_1_02_wcrjr_1__12__02_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcrjr_1__12__03_P':hp.choice('_1_02_wcrjr_1__12__03_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcrjr_1__12__04_INT_O':hp.quniform('_1_02_wcrjr_1__12__04_INT_O', 1.0, 5.0,1.0), + '_1_02_wcrjr_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wcror_0_QUOTE_START_B':'weka.classifiers.rules.OneR', + '_1_02_wcror_1__13__01_INT_B':hp.qloguniform('_1_02_wcror_1__13__01_INT_B', math.log(1.0), math.log(32.0), 1.0), + '_1_02_wcror_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wcrpart_0_QUOTE_START_B':'weka.classifiers.rules.PART', + '_1_02_wcrpart_1__14__01_INT_N':hp.quniform('_1_02_wcrpart_1__14__01_INT_N', 2.0, 5.0,1.0), + '_1_02_wcrpart_1__14__02_INT_M':hp.qloguniform('_1_02_wcrpart_1__14__02_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_02_wcrpart_1__14__03_R':hp.choice('_1_02_wcrpart_1__14__03_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcrpart_1__14__04_B':hp.choice('_1_02_wcrpart_1__14__04_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wcrpart_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wcrzr_0_QUOTE_START_B':'weka.classifiers.rules.ZeroR', + '_1_02_wcrzr_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wctds_0_QUOTE_START_B':'weka.classifiers.trees.DecisionStump', + '_1_02_wctds_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wctj_0_QUOTE_START_B':'weka.classifiers.trees.J48', + '_1_02_wctj_1__17__01_O':hp.choice('_1_02_wctj_1__17__01_O', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wctj_1__17__02_U':hp.choice('_1_02_wctj_1__17__02_U', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wctj_1__17__03_B':hp.choice('_1_02_wctj_1__17__03_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wctj_1__17__04_J':hp.choice('_1_02_wctj_1__17__04_J', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wctj_1__17__05_A':hp.choice('_1_02_wctj_1__17__05_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wctj_1__17__06_S':hp.choice('_1_02_wctj_1__17__06_S', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wctj_1__17__07_INT_M':hp.qloguniform('_1_02_wctj_1__17__07_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_02_wctj_1__17__08_C':hp.uniform('_1_02_wctj_1__17__08_C', 0.0, 1.0), + '_1_02_wctj_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wctlmt_0_QUOTE_START_B':'weka.classifiers.trees.LMT', + '_1_02_wctlmt_1__18__01_B':hp.choice('_1_02_wctlmt_1__18__01_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wctlmt_1__18__02_R':hp.choice('_1_02_wctlmt_1__18__02_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wctlmt_1__18__03_C':hp.choice('_1_02_wctlmt_1__18__03_C', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wctlmt_1__18__04_P':hp.choice('_1_02_wctlmt_1__18__04_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wctlmt_1__18__05_INT_M':hp.qloguniform('_1_02_wctlmt_1__18__05_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_02_wctlmt_1__18__06_A':hp.choice('_1_02_wctlmt_1__18__06_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wctlmt_1__18__07_auto_param_291':hp.choice('_1_02_wctlmt_1__18__07_auto_param_291', [ + { + '_1_02_wctlmt_1__18__07__00__00_W_HIDDEN':'0', + '_1_02_wctlmt_1__18__07__00__01_1_W':hp.choice('_1_02_wctlmt_1__18__07__00__01_1_W', ['0', ]), + }, + { + '_1_02_wctlmt_1__18__07__01__00_W_HIDDEN':'1', + '_1_02_wctlmt_1__18__07__01__01_2_W':hp.uniform('_1_02_wctlmt_1__18__07__01__01_2_W', 0.0, 1.0), + }, + ]), + '_1_02_wctlmt_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wctrept_0_QUOTE_START_B':'weka.classifiers.trees.REPTree', + '_1_02_wctrept_1__19__01_INT_M':hp.qloguniform('_1_02_wctrept_1__19__01_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_02_wctrept_1__19__02_V':hp.loguniform('_1_02_wctrept_1__19__02_V', math.log(1.0E-5), math.log(0.1)), + '_1_02_wctrept_1__19__03_P':hp.choice('_1_02_wctrept_1__19__03_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wctrept_1__19__04_auto_param_295':hp.choice('_1_02_wctrept_1__19__04_auto_param_295', [ + { + '_1_02_wctrept_1__19__04__00__00_depth_HIDDEN':'0', + '_1_02_wctrept_1__19__04__00__01_1_INT_L':hp.choice('_1_02_wctrept_1__19__04__00__01_1_INT_L', ['-1', ]), + }, + { + '_1_02_wctrept_1__19__04__01__00_depth_HIDDEN':'1', + '_1_02_wctrept_1__19__04__01__01_2_INT_L':hp.quniform('_1_02_wctrept_1__19__04__01__01_2_INT_L', 2.0, 20.0,1.0), + }, + ]), + '_1_02_wctrept_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wctrf_0_QUOTE_START_B':'weka.classifiers.trees.RandomForest', + '_1_02_wctrf_1__20__01_INT_I':hp.qloguniform('_1_02_wctrf_1__20__01_INT_I', math.log(2.0), math.log(256.0), 1.0), + '_1_02_wctrf_1__20__02_auto_param_299':hp.choice('_1_02_wctrf_1__20__02_auto_param_299', [ + { + '_1_02_wctrf_1__20__02__00__00_features_HIDDEN':'0', + '_1_02_wctrf_1__20__02__00__01_1_INT_K':hp.choice('_1_02_wctrf_1__20__02__00__01_1_INT_K', ['1', ]), + }, + { + '_1_02_wctrf_1__20__02__01__00_features_HIDDEN':'1', + '_1_02_wctrf_1__20__02__01__01_2_INT_K':hp.qloguniform('_1_02_wctrf_1__20__02__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0), + }, + ]), + '_1_02_wctrf_1__20__03_auto_param_302':hp.choice('_1_02_wctrf_1__20__03_auto_param_302', [ + { + '_1_02_wctrf_1__20__03__00__00_depth_HIDDEN':'0', + '_1_02_wctrf_1__20__03__00__01_1_INT_depth':hp.choice('_1_02_wctrf_1__20__03__00__01_1_INT_depth', ['1', ]), + }, + { + '_1_02_wctrf_1__20__03__01__00_depth_HIDDEN':'1', + '_1_02_wctrf_1__20__03__01__01_2_INT_depth':hp.quniform('_1_02_wctrf_1__20__03__01__01_2_INT_depth', 2.0, 20.0,1.0), + }, + ]), + '_1_02_wctrf_2_QUOTE_END':'REMOVED', + }, + { + '_1_02_wctrt_0_QUOTE_START_B':'weka.classifiers.trees.RandomTree', + '_1_02_wctrt_1__21__01_INT_M':hp.qloguniform('_1_02_wctrt_1__21__01_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_02_wctrt_1__21__02_U':hp.choice('_1_02_wctrt_1__21__02_U', ['REMOVE_PREV', 'REMOVED', ]), + '_1_02_wctrt_1__21__03_auto_param_306':hp.choice('_1_02_wctrt_1__21__03_auto_param_306', [ + { + '_1_02_wctrt_1__21__03__00__00_features_HIDDEN':'0', + '_1_02_wctrt_1__21__03__00__01_1_INT_K':hp.choice('_1_02_wctrt_1__21__03__00__01_1_INT_K', ['0', ]), + }, + { + '_1_02_wctrt_1__21__03__01__00_features_HIDDEN':'1', + '_1_02_wctrt_1__21__03__01__01_2_INT_K':hp.qloguniform('_1_02_wctrt_1__21__03__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0), + }, + ]), + '_1_02_wctrt_1__21__04_auto_param_309':hp.choice('_1_02_wctrt_1__21__04_auto_param_309', [ + { + '_1_02_wctrt_1__21__04__00__00_depth_HIDDEN':'0', + '_1_02_wctrt_1__21__04__00__01_1_INT_depth':hp.choice('_1_02_wctrt_1__21__04__00__01_1_INT_depth', ['0', ]), + }, + { + '_1_02_wctrt_1__21__04__01__00_depth_HIDDEN':'1', + '_1_02_wctrt_1__21__04__01__01_2_INT_depth':hp.quniform('_1_02_wctrt_1__21__04__01__01_2_INT_depth', 2.0, 20.0,1.0), + }, + ]), + '_1_02_wctrt_1__21__05_auto_param_312':hp.choice('_1_02_wctrt_1__21__05_auto_param_312', [ + { + '_1_02_wctrt_1__21__05__00__00_back_HIDDEN':'0', + '_1_02_wctrt_1__21__05__00__01_1_INT_N':hp.choice('_1_02_wctrt_1__21__05__00__01_1_INT_N', ['0', ]), + }, + { + '_1_02_wctrt_1__21__05__01__00_back_HIDDEN':'1', + '_1_02_wctrt_1__21__05__01__01_2_INT_N':hp.quniform('_1_02_wctrt_1__21__05__01__01_2_INT_N', 2.0, 5.0,1.0), + }, + ]), + '_1_02_wctrt_2_QUOTE_END':'REMOVED', + }, + ]), + hp.choice('auto_param_315', [ + ( + ), + ( + hp.choice('auto_param_318', [ + { + '_1_03_wcbbn_0_QUOTE_START_B':'weka.classifiers.bayes.BayesNet', + '_1_03_wcbbn_1__00__01_D':hp.choice('_1_03_wcbbn_1__00__01_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcbbn_1__00__02_Q':hp.choice('_1_03_wcbbn_1__00__02_Q', ['weka.classifiers.bayes.net.search.local.TAN', 'weka.classifiers.bayes.net.search.local.HillClimber', 'weka.classifiers.bayes.net.search.local.LAGDHillClimber', 'weka.classifiers.bayes.net.search.local.SimulatedAnnealing', 'weka.classifiers.bayes.net.search.local.TabuSearch', 'weka.classifiers.bayes.net.search.local.K2', ]), + '_1_03_wcbbn_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wcbnb_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayes', + '_1_03_wcbnb_1__01__01_K':hp.choice('_1_03_wcbnb_1__01__01_K', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcbnb_1__01__02_D':hp.choice('_1_03_wcbnb_1__01__02_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcbnb_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wcbnbm_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayesMultinomial', + '_1_03_wcbnbm_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wcfl_0_QUOTE_START_B':'weka.classifiers.functions.Logistic', + '_1_03_wcfl_1__03__01_R':hp.loguniform('_1_03_wcfl_1__03__01_R', math.log(1.0E-12), math.log(10.0)), + '_1_03_wcfl_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wcfmp_0_QUOTE_START_B':'weka.classifiers.functions.MultilayerPerceptron', + '_1_03_wcfmp_1__04__01_L':hp.uniform('_1_03_wcfmp_1__04__01_L', 0.1, 1.0), + '_1_03_wcfmp_1__04__02_M':hp.uniform('_1_03_wcfmp_1__04__02_M', 0.1, 1.0), + '_1_03_wcfmp_1__04__03_B':hp.choice('_1_03_wcfmp_1__04__03_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcfmp_1__04__04_H':hp.choice('_1_03_wcfmp_1__04__04_H', ['t', 'i', 'o', 'a', ]), + '_1_03_wcfmp_1__04__05_C':hp.choice('_1_03_wcfmp_1__04__05_C', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcfmp_1__04__06_R':hp.choice('_1_03_wcfmp_1__04__06_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcfmp_1__04__07_D':hp.choice('_1_03_wcfmp_1__04__07_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcfmp_1__04__08_S':hp.choice('_1_03_wcfmp_1__04__08_S', ['1', ]), + '_1_03_wcfmp_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wcfsgd_0_QUOTE_START_B':'weka.classifiers.functions.SGD', + '_1_03_wcfsgd_1__05__01_F':hp.choice('_1_03_wcfsgd_1__05__01_F', ['2', '1', '0', ]), + '_1_03_wcfsgd_1__05__02_L':hp.loguniform('_1_03_wcfsgd_1__05__02_L', math.log(1.0E-5), math.log(0.1)), + '_1_03_wcfsgd_1__05__03_R':hp.loguniform('_1_03_wcfsgd_1__05__03_R', math.log(1.0E-12), math.log(10.0)), + '_1_03_wcfsgd_1__05__04_N':hp.choice('_1_03_wcfsgd_1__05__04_N', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcfsgd_1__05__05_M':hp.choice('_1_03_wcfsgd_1__05__05_M', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcfsgd_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wcfsmo_0_QUOTE_START_B':'weka.classifiers.functions.SMO', + '_1_03_wcfsmo_1__06__01_0_C':hp.uniform('_1_03_wcfsmo_1__06__01_0_C', 0.5, 1.5), + '_1_03_wcfsmo_1__06__02_1_N':hp.choice('_1_03_wcfsmo_1__06__02_1_N', ['2', '1', '0', ]), + '_1_03_wcfsmo_1__06__03_2_M':hp.choice('_1_03_wcfsmo_1__06__03_2_M', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcfsmo_1__06__04_5_QUOTE_END':hp.choice('_1_03_wcfsmo_1__06__04_5_QUOTE_END', ['REMOVED', ]), + '_1_03_wcfsmo_1__06__05_auto_param_326':hp.choice('_1_03_wcfsmo_1__06__05_auto_param_326', [ + { + '_1_03_wcfsmo_1__06__05__00__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.NormalizedPolyKernel', + '_1_03_wcfsmo_1__06__05__00__01_4_npoly_E':hp.uniform('_1_03_wcfsmo_1__06__05__00__01_4_npoly_E', 0.2, 5.0), + '_1_03_wcfsmo_1__06__05__00__02_4_npoly_L':hp.choice('_1_03_wcfsmo_1__06__05__00__02_4_npoly_L', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_03_wcfsmo_1__06__05__01__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.PolyKernel', + '_1_03_wcfsmo_1__06__05__01__01_4_poly_E':hp.uniform('_1_03_wcfsmo_1__06__05__01__01_4_poly_E', 0.2, 5.0), + '_1_03_wcfsmo_1__06__05__01__02_4_poly_L':hp.choice('_1_03_wcfsmo_1__06__05__01__02_4_poly_L', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_03_wcfsmo_1__06__05__02__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.Puk', + '_1_03_wcfsmo_1__06__05__02__01_4_puk_S':hp.uniform('_1_03_wcfsmo_1__06__05__02__01_4_puk_S', 0.1, 10.0), + '_1_03_wcfsmo_1__06__05__02__02_4_puk_O':hp.uniform('_1_03_wcfsmo_1__06__05__02__02_4_puk_O', 0.1, 1.0), + }, + { + '_1_03_wcfsmo_1__06__05__03__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.RBFKernel', + '_1_03_wcfsmo_1__06__05__03__01_4_rbf_G':hp.loguniform('_1_03_wcfsmo_1__06__05__03__01_4_rbf_G', math.log(1.0E-4), math.log(1.0)), + }, + ]), + '_1_03_wcfsmo_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wcfsl_0_QUOTE_START_B':'weka.classifiers.functions.SimpleLogistic', + '_1_03_wcfsl_1__07__01_S':hp.choice('_1_03_wcfsl_1__07__01_S', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcfsl_1__07__02_A':hp.choice('_1_03_wcfsl_1__07__02_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcfsl_1__07__03_auto_param_332':hp.choice('_1_03_wcfsl_1__07__03_auto_param_332', [ + { + '_1_03_wcfsl_1__07__03__00__00_W_HIDDEN':'0', + '_1_03_wcfsl_1__07__03__00__01_1_W':hp.choice('_1_03_wcfsl_1__07__03__00__01_1_W', ['0', ]), + }, + { + '_1_03_wcfsl_1__07__03__01__00_W_HIDDEN':'1', + '_1_03_wcfsl_1__07__03__01__01_2_W':hp.uniform('_1_03_wcfsl_1__07__03__01__01_2_W', 0.0, 1.0), + }, + ]), + '_1_03_wcfsl_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wcfvp_0_QUOTE_START_B':'weka.classifiers.functions.VotedPerceptron', + '_1_03_wcfvp_1__08__01_INT_I':hp.quniform('_1_03_wcfvp_1__08__01_INT_I', 1.0, 10.0,1.0), + '_1_03_wcfvp_1__08__02_INT_M':hp.qloguniform('_1_03_wcfvp_1__08__02_INT_M', math.log(5000.0), math.log(50000.0), 1.0), + '_1_03_wcfvp_1__08__03_E':hp.uniform('_1_03_wcfvp_1__08__03_E', 0.2, 5.0), + '_1_03_wcfvp_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wclib_0_QUOTE_START_B':'weka.classifiers.lazy.IBk', + '_1_03_wclib_1__09__01_E':hp.choice('_1_03_wclib_1__09__01_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wclib_1__09__02_INT_K':hp.qloguniform('_1_03_wclib_1__09__02_INT_K', math.log(1.0), math.log(64.0), 1.0), + '_1_03_wclib_1__09__03_X':hp.choice('_1_03_wclib_1__09__03_X', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wclib_1__09__04_F':hp.choice('_1_03_wclib_1__09__04_F', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wclib_1__09__05_I':hp.choice('_1_03_wclib_1__09__05_I', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wclib_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wclks_0_QUOTE_START_B':'weka.classifiers.lazy.KStar', + '_1_03_wclks_1__10__01_INT_B':hp.quniform('_1_03_wclks_1__10__01_INT_B', 1.0, 100.0,1.0), + '_1_03_wclks_1__10__02_E':hp.choice('_1_03_wclks_1__10__02_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wclks_1__10__03_M':hp.choice('_1_03_wclks_1__10__03_M', ['n', 'd', 'm', 'a', ]), + '_1_03_wclks_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wcrdt_0_QUOTE_START_B':'weka.classifiers.rules.DecisionTable', + '_1_03_wcrdt_1__11__01_E':hp.choice('_1_03_wcrdt_1__11__01_E', ['auc', 'rmse', 'mae', 'acc', ]), + '_1_03_wcrdt_1__11__02_I':hp.choice('_1_03_wcrdt_1__11__02_I', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcrdt_1__11__03_S':hp.choice('_1_03_wcrdt_1__11__03_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]), + '_1_03_wcrdt_1__11__04_X':hp.choice('_1_03_wcrdt_1__11__04_X', ['4', '2', '3', '1', ]), + '_1_03_wcrdt_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wcrjr_0_QUOTE_START_B':'weka.classifiers.rules.JRip', + '_1_03_wcrjr_1__12__01_N':hp.uniform('_1_03_wcrjr_1__12__01_N', 1.0, 5.0), + '_1_03_wcrjr_1__12__02_E':hp.choice('_1_03_wcrjr_1__12__02_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcrjr_1__12__03_P':hp.choice('_1_03_wcrjr_1__12__03_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcrjr_1__12__04_INT_O':hp.quniform('_1_03_wcrjr_1__12__04_INT_O', 1.0, 5.0,1.0), + '_1_03_wcrjr_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wcror_0_QUOTE_START_B':'weka.classifiers.rules.OneR', + '_1_03_wcror_1__13__01_INT_B':hp.qloguniform('_1_03_wcror_1__13__01_INT_B', math.log(1.0), math.log(32.0), 1.0), + '_1_03_wcror_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wcrpart_0_QUOTE_START_B':'weka.classifiers.rules.PART', + '_1_03_wcrpart_1__14__01_INT_N':hp.quniform('_1_03_wcrpart_1__14__01_INT_N', 2.0, 5.0,1.0), + '_1_03_wcrpart_1__14__02_INT_M':hp.qloguniform('_1_03_wcrpart_1__14__02_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_03_wcrpart_1__14__03_R':hp.choice('_1_03_wcrpart_1__14__03_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcrpart_1__14__04_B':hp.choice('_1_03_wcrpart_1__14__04_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wcrpart_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wcrzr_0_QUOTE_START_B':'weka.classifiers.rules.ZeroR', + '_1_03_wcrzr_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wctds_0_QUOTE_START_B':'weka.classifiers.trees.DecisionStump', + '_1_03_wctds_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wctj_0_QUOTE_START_B':'weka.classifiers.trees.J48', + '_1_03_wctj_1__17__01_O':hp.choice('_1_03_wctj_1__17__01_O', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wctj_1__17__02_U':hp.choice('_1_03_wctj_1__17__02_U', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wctj_1__17__03_B':hp.choice('_1_03_wctj_1__17__03_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wctj_1__17__04_J':hp.choice('_1_03_wctj_1__17__04_J', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wctj_1__17__05_A':hp.choice('_1_03_wctj_1__17__05_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wctj_1__17__06_S':hp.choice('_1_03_wctj_1__17__06_S', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wctj_1__17__07_INT_M':hp.qloguniform('_1_03_wctj_1__17__07_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_03_wctj_1__17__08_C':hp.uniform('_1_03_wctj_1__17__08_C', 0.0, 1.0), + '_1_03_wctj_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wctlmt_0_QUOTE_START_B':'weka.classifiers.trees.LMT', + '_1_03_wctlmt_1__18__01_B':hp.choice('_1_03_wctlmt_1__18__01_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wctlmt_1__18__02_R':hp.choice('_1_03_wctlmt_1__18__02_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wctlmt_1__18__03_C':hp.choice('_1_03_wctlmt_1__18__03_C', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wctlmt_1__18__04_P':hp.choice('_1_03_wctlmt_1__18__04_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wctlmt_1__18__05_INT_M':hp.qloguniform('_1_03_wctlmt_1__18__05_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_03_wctlmt_1__18__06_A':hp.choice('_1_03_wctlmt_1__18__06_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wctlmt_1__18__07_auto_param_346':hp.choice('_1_03_wctlmt_1__18__07_auto_param_346', [ + { + '_1_03_wctlmt_1__18__07__00__00_W_HIDDEN':'0', + '_1_03_wctlmt_1__18__07__00__01_1_W':hp.choice('_1_03_wctlmt_1__18__07__00__01_1_W', ['0', ]), + }, + { + '_1_03_wctlmt_1__18__07__01__00_W_HIDDEN':'1', + '_1_03_wctlmt_1__18__07__01__01_2_W':hp.uniform('_1_03_wctlmt_1__18__07__01__01_2_W', 0.0, 1.0), + }, + ]), + '_1_03_wctlmt_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wctrept_0_QUOTE_START_B':'weka.classifiers.trees.REPTree', + '_1_03_wctrept_1__19__01_INT_M':hp.qloguniform('_1_03_wctrept_1__19__01_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_03_wctrept_1__19__02_V':hp.loguniform('_1_03_wctrept_1__19__02_V', math.log(1.0E-5), math.log(0.1)), + '_1_03_wctrept_1__19__03_P':hp.choice('_1_03_wctrept_1__19__03_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wctrept_1__19__04_auto_param_350':hp.choice('_1_03_wctrept_1__19__04_auto_param_350', [ + { + '_1_03_wctrept_1__19__04__00__00_depth_HIDDEN':'0', + '_1_03_wctrept_1__19__04__00__01_1_INT_L':hp.choice('_1_03_wctrept_1__19__04__00__01_1_INT_L', ['-1', ]), + }, + { + '_1_03_wctrept_1__19__04__01__00_depth_HIDDEN':'1', + '_1_03_wctrept_1__19__04__01__01_2_INT_L':hp.quniform('_1_03_wctrept_1__19__04__01__01_2_INT_L', 2.0, 20.0,1.0), + }, + ]), + '_1_03_wctrept_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wctrf_0_QUOTE_START_B':'weka.classifiers.trees.RandomForest', + '_1_03_wctrf_1__20__01_INT_I':hp.qloguniform('_1_03_wctrf_1__20__01_INT_I', math.log(2.0), math.log(256.0), 1.0), + '_1_03_wctrf_1__20__02_auto_param_354':hp.choice('_1_03_wctrf_1__20__02_auto_param_354', [ + { + '_1_03_wctrf_1__20__02__00__00_features_HIDDEN':'0', + '_1_03_wctrf_1__20__02__00__01_1_INT_K':hp.choice('_1_03_wctrf_1__20__02__00__01_1_INT_K', ['1', ]), + }, + { + '_1_03_wctrf_1__20__02__01__00_features_HIDDEN':'1', + '_1_03_wctrf_1__20__02__01__01_2_INT_K':hp.qloguniform('_1_03_wctrf_1__20__02__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0), + }, + ]), + '_1_03_wctrf_1__20__03_auto_param_357':hp.choice('_1_03_wctrf_1__20__03_auto_param_357', [ + { + '_1_03_wctrf_1__20__03__00__00_depth_HIDDEN':'0', + '_1_03_wctrf_1__20__03__00__01_1_INT_depth':hp.choice('_1_03_wctrf_1__20__03__00__01_1_INT_depth', ['1', ]), + }, + { + '_1_03_wctrf_1__20__03__01__00_depth_HIDDEN':'1', + '_1_03_wctrf_1__20__03__01__01_2_INT_depth':hp.quniform('_1_03_wctrf_1__20__03__01__01_2_INT_depth', 2.0, 20.0,1.0), + }, + ]), + '_1_03_wctrf_2_QUOTE_END':'REMOVED', + }, + { + '_1_03_wctrt_0_QUOTE_START_B':'weka.classifiers.trees.RandomTree', + '_1_03_wctrt_1__21__01_INT_M':hp.qloguniform('_1_03_wctrt_1__21__01_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_03_wctrt_1__21__02_U':hp.choice('_1_03_wctrt_1__21__02_U', ['REMOVE_PREV', 'REMOVED', ]), + '_1_03_wctrt_1__21__03_auto_param_361':hp.choice('_1_03_wctrt_1__21__03_auto_param_361', [ + { + '_1_03_wctrt_1__21__03__00__00_features_HIDDEN':'0', + '_1_03_wctrt_1__21__03__00__01_1_INT_K':hp.choice('_1_03_wctrt_1__21__03__00__01_1_INT_K', ['0', ]), + }, + { + '_1_03_wctrt_1__21__03__01__00_features_HIDDEN':'1', + '_1_03_wctrt_1__21__03__01__01_2_INT_K':hp.qloguniform('_1_03_wctrt_1__21__03__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0), + }, + ]), + '_1_03_wctrt_1__21__04_auto_param_364':hp.choice('_1_03_wctrt_1__21__04_auto_param_364', [ + { + '_1_03_wctrt_1__21__04__00__00_depth_HIDDEN':'0', + '_1_03_wctrt_1__21__04__00__01_1_INT_depth':hp.choice('_1_03_wctrt_1__21__04__00__01_1_INT_depth', ['0', ]), + }, + { + '_1_03_wctrt_1__21__04__01__00_depth_HIDDEN':'1', + '_1_03_wctrt_1__21__04__01__01_2_INT_depth':hp.quniform('_1_03_wctrt_1__21__04__01__01_2_INT_depth', 2.0, 20.0,1.0), + }, + ]), + '_1_03_wctrt_1__21__05_auto_param_367':hp.choice('_1_03_wctrt_1__21__05_auto_param_367', [ + { + '_1_03_wctrt_1__21__05__00__00_back_HIDDEN':'0', + '_1_03_wctrt_1__21__05__00__01_1_INT_N':hp.choice('_1_03_wctrt_1__21__05__00__01_1_INT_N', ['0', ]), + }, + { + '_1_03_wctrt_1__21__05__01__00_back_HIDDEN':'1', + '_1_03_wctrt_1__21__05__01__01_2_INT_N':hp.quniform('_1_03_wctrt_1__21__05__01__01_2_INT_N', 2.0, 5.0,1.0), + }, + ]), + '_1_03_wctrt_2_QUOTE_END':'REMOVED', + }, + ]), + hp.choice('auto_param_370', [ + ( + ), + ( + hp.choice('auto_param_373', [ + { + '_1_04_wcbbn_0_QUOTE_START_B':'weka.classifiers.bayes.BayesNet', + '_1_04_wcbbn_1__00__01_D':hp.choice('_1_04_wcbbn_1__00__01_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcbbn_1__00__02_Q':hp.choice('_1_04_wcbbn_1__00__02_Q', ['weka.classifiers.bayes.net.search.local.TAN', 'weka.classifiers.bayes.net.search.local.HillClimber', 'weka.classifiers.bayes.net.search.local.LAGDHillClimber', 'weka.classifiers.bayes.net.search.local.SimulatedAnnealing', 'weka.classifiers.bayes.net.search.local.TabuSearch', 'weka.classifiers.bayes.net.search.local.K2', ]), + '_1_04_wcbbn_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wcbnb_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayes', + '_1_04_wcbnb_1__01__01_K':hp.choice('_1_04_wcbnb_1__01__01_K', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcbnb_1__01__02_D':hp.choice('_1_04_wcbnb_1__01__02_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcbnb_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wcbnbm_0_QUOTE_START_B':'weka.classifiers.bayes.NaiveBayesMultinomial', + '_1_04_wcbnbm_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wcfl_0_QUOTE_START_B':'weka.classifiers.functions.Logistic', + '_1_04_wcfl_1__03__01_R':hp.loguniform('_1_04_wcfl_1__03__01_R', math.log(1.0E-12), math.log(10.0)), + '_1_04_wcfl_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wcfmp_0_QUOTE_START_B':'weka.classifiers.functions.MultilayerPerceptron', + '_1_04_wcfmp_1__04__01_L':hp.uniform('_1_04_wcfmp_1__04__01_L', 0.1, 1.0), + '_1_04_wcfmp_1__04__02_M':hp.uniform('_1_04_wcfmp_1__04__02_M', 0.1, 1.0), + '_1_04_wcfmp_1__04__03_B':hp.choice('_1_04_wcfmp_1__04__03_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcfmp_1__04__04_H':hp.choice('_1_04_wcfmp_1__04__04_H', ['t', 'i', 'o', 'a', ]), + '_1_04_wcfmp_1__04__05_C':hp.choice('_1_04_wcfmp_1__04__05_C', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcfmp_1__04__06_R':hp.choice('_1_04_wcfmp_1__04__06_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcfmp_1__04__07_D':hp.choice('_1_04_wcfmp_1__04__07_D', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcfmp_1__04__08_S':hp.choice('_1_04_wcfmp_1__04__08_S', ['1', ]), + '_1_04_wcfmp_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wcfsgd_0_QUOTE_START_B':'weka.classifiers.functions.SGD', + '_1_04_wcfsgd_1__05__01_F':hp.choice('_1_04_wcfsgd_1__05__01_F', ['2', '1', '0', ]), + '_1_04_wcfsgd_1__05__02_L':hp.loguniform('_1_04_wcfsgd_1__05__02_L', math.log(1.0E-5), math.log(0.1)), + '_1_04_wcfsgd_1__05__03_R':hp.loguniform('_1_04_wcfsgd_1__05__03_R', math.log(1.0E-12), math.log(10.0)), + '_1_04_wcfsgd_1__05__04_N':hp.choice('_1_04_wcfsgd_1__05__04_N', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcfsgd_1__05__05_M':hp.choice('_1_04_wcfsgd_1__05__05_M', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcfsgd_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wcfsmo_0_QUOTE_START_B':'weka.classifiers.functions.SMO', + '_1_04_wcfsmo_1__06__01_0_C':hp.uniform('_1_04_wcfsmo_1__06__01_0_C', 0.5, 1.5), + '_1_04_wcfsmo_1__06__02_1_N':hp.choice('_1_04_wcfsmo_1__06__02_1_N', ['2', '1', '0', ]), + '_1_04_wcfsmo_1__06__03_2_M':hp.choice('_1_04_wcfsmo_1__06__03_2_M', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcfsmo_1__06__04_5_QUOTE_END':hp.choice('_1_04_wcfsmo_1__06__04_5_QUOTE_END', ['REMOVED', ]), + '_1_04_wcfsmo_1__06__05_auto_param_381':hp.choice('_1_04_wcfsmo_1__06__05_auto_param_381', [ + { + '_1_04_wcfsmo_1__06__05__00__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.NormalizedPolyKernel', + '_1_04_wcfsmo_1__06__05__00__01_4_npoly_E':hp.uniform('_1_04_wcfsmo_1__06__05__00__01_4_npoly_E', 0.2, 5.0), + '_1_04_wcfsmo_1__06__05__00__02_4_npoly_L':hp.choice('_1_04_wcfsmo_1__06__05__00__02_4_npoly_L', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_04_wcfsmo_1__06__05__01__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.PolyKernel', + '_1_04_wcfsmo_1__06__05__01__01_4_poly_E':hp.uniform('_1_04_wcfsmo_1__06__05__01__01_4_poly_E', 0.2, 5.0), + '_1_04_wcfsmo_1__06__05__01__02_4_poly_L':hp.choice('_1_04_wcfsmo_1__06__05__01__02_4_poly_L', ['REMOVE_PREV', 'REMOVED', ]), + }, + { + '_1_04_wcfsmo_1__06__05__02__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.Puk', + '_1_04_wcfsmo_1__06__05__02__01_4_puk_S':hp.uniform('_1_04_wcfsmo_1__06__05__02__01_4_puk_S', 0.1, 10.0), + '_1_04_wcfsmo_1__06__05__02__02_4_puk_O':hp.uniform('_1_04_wcfsmo_1__06__05__02__02_4_puk_O', 0.1, 1.0), + }, + { + '_1_04_wcfsmo_1__06__05__03__00_3_REG_IGNORE_QUOTE_START_K':'weka.classifiers.functions.supportVector.RBFKernel', + '_1_04_wcfsmo_1__06__05__03__01_4_rbf_G':hp.loguniform('_1_04_wcfsmo_1__06__05__03__01_4_rbf_G', math.log(1.0E-4), math.log(1.0)), + }, + ]), + '_1_04_wcfsmo_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wcfsl_0_QUOTE_START_B':'weka.classifiers.functions.SimpleLogistic', + '_1_04_wcfsl_1__07__01_S':hp.choice('_1_04_wcfsl_1__07__01_S', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcfsl_1__07__02_A':hp.choice('_1_04_wcfsl_1__07__02_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcfsl_1__07__03_auto_param_387':hp.choice('_1_04_wcfsl_1__07__03_auto_param_387', [ + { + '_1_04_wcfsl_1__07__03__00__00_W_HIDDEN':'0', + '_1_04_wcfsl_1__07__03__00__01_1_W':hp.choice('_1_04_wcfsl_1__07__03__00__01_1_W', ['0', ]), + }, + { + '_1_04_wcfsl_1__07__03__01__00_W_HIDDEN':'1', + '_1_04_wcfsl_1__07__03__01__01_2_W':hp.uniform('_1_04_wcfsl_1__07__03__01__01_2_W', 0.0, 1.0), + }, + ]), + '_1_04_wcfsl_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wcfvp_0_QUOTE_START_B':'weka.classifiers.functions.VotedPerceptron', + '_1_04_wcfvp_1__08__01_INT_I':hp.quniform('_1_04_wcfvp_1__08__01_INT_I', 1.0, 10.0,1.0), + '_1_04_wcfvp_1__08__02_INT_M':hp.qloguniform('_1_04_wcfvp_1__08__02_INT_M', math.log(5000.0), math.log(50000.0), 1.0), + '_1_04_wcfvp_1__08__03_E':hp.uniform('_1_04_wcfvp_1__08__03_E', 0.2, 5.0), + '_1_04_wcfvp_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wclib_0_QUOTE_START_B':'weka.classifiers.lazy.IBk', + '_1_04_wclib_1__09__01_E':hp.choice('_1_04_wclib_1__09__01_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wclib_1__09__02_INT_K':hp.qloguniform('_1_04_wclib_1__09__02_INT_K', math.log(1.0), math.log(64.0), 1.0), + '_1_04_wclib_1__09__03_X':hp.choice('_1_04_wclib_1__09__03_X', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wclib_1__09__04_F':hp.choice('_1_04_wclib_1__09__04_F', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wclib_1__09__05_I':hp.choice('_1_04_wclib_1__09__05_I', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wclib_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wclks_0_QUOTE_START_B':'weka.classifiers.lazy.KStar', + '_1_04_wclks_1__10__01_INT_B':hp.quniform('_1_04_wclks_1__10__01_INT_B', 1.0, 100.0,1.0), + '_1_04_wclks_1__10__02_E':hp.choice('_1_04_wclks_1__10__02_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wclks_1__10__03_M':hp.choice('_1_04_wclks_1__10__03_M', ['n', 'd', 'm', 'a', ]), + '_1_04_wclks_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wcrdt_0_QUOTE_START_B':'weka.classifiers.rules.DecisionTable', + '_1_04_wcrdt_1__11__01_E':hp.choice('_1_04_wcrdt_1__11__01_E', ['auc', 'rmse', 'mae', 'acc', ]), + '_1_04_wcrdt_1__11__02_I':hp.choice('_1_04_wcrdt_1__11__02_I', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcrdt_1__11__03_S':hp.choice('_1_04_wcrdt_1__11__03_S', ['weka.attributeSelection.Ranker', 'weka.attributeSelection.GreedyStepwise', 'weka.attributeSelection.BestFirst', ]), + '_1_04_wcrdt_1__11__04_X':hp.choice('_1_04_wcrdt_1__11__04_X', ['4', '2', '3', '1', ]), + '_1_04_wcrdt_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wcrjr_0_QUOTE_START_B':'weka.classifiers.rules.JRip', + '_1_04_wcrjr_1__12__01_N':hp.uniform('_1_04_wcrjr_1__12__01_N', 1.0, 5.0), + '_1_04_wcrjr_1__12__02_E':hp.choice('_1_04_wcrjr_1__12__02_E', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcrjr_1__12__03_P':hp.choice('_1_04_wcrjr_1__12__03_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcrjr_1__12__04_INT_O':hp.quniform('_1_04_wcrjr_1__12__04_INT_O', 1.0, 5.0,1.0), + '_1_04_wcrjr_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wcror_0_QUOTE_START_B':'weka.classifiers.rules.OneR', + '_1_04_wcror_1__13__01_INT_B':hp.qloguniform('_1_04_wcror_1__13__01_INT_B', math.log(1.0), math.log(32.0), 1.0), + '_1_04_wcror_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wcrpart_0_QUOTE_START_B':'weka.classifiers.rules.PART', + '_1_04_wcrpart_1__14__01_INT_N':hp.quniform('_1_04_wcrpart_1__14__01_INT_N', 2.0, 5.0,1.0), + '_1_04_wcrpart_1__14__02_INT_M':hp.qloguniform('_1_04_wcrpart_1__14__02_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_04_wcrpart_1__14__03_R':hp.choice('_1_04_wcrpart_1__14__03_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcrpart_1__14__04_B':hp.choice('_1_04_wcrpart_1__14__04_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wcrpart_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wcrzr_0_QUOTE_START_B':'weka.classifiers.rules.ZeroR', + '_1_04_wcrzr_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wctds_0_QUOTE_START_B':'weka.classifiers.trees.DecisionStump', + '_1_04_wctds_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wctj_0_QUOTE_START_B':'weka.classifiers.trees.J48', + '_1_04_wctj_1__17__01_O':hp.choice('_1_04_wctj_1__17__01_O', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wctj_1__17__02_U':hp.choice('_1_04_wctj_1__17__02_U', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wctj_1__17__03_B':hp.choice('_1_04_wctj_1__17__03_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wctj_1__17__04_J':hp.choice('_1_04_wctj_1__17__04_J', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wctj_1__17__05_A':hp.choice('_1_04_wctj_1__17__05_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wctj_1__17__06_S':hp.choice('_1_04_wctj_1__17__06_S', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wctj_1__17__07_INT_M':hp.qloguniform('_1_04_wctj_1__17__07_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_04_wctj_1__17__08_C':hp.uniform('_1_04_wctj_1__17__08_C', 0.0, 1.0), + '_1_04_wctj_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wctlmt_0_QUOTE_START_B':'weka.classifiers.trees.LMT', + '_1_04_wctlmt_1__18__01_B':hp.choice('_1_04_wctlmt_1__18__01_B', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wctlmt_1__18__02_R':hp.choice('_1_04_wctlmt_1__18__02_R', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wctlmt_1__18__03_C':hp.choice('_1_04_wctlmt_1__18__03_C', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wctlmt_1__18__04_P':hp.choice('_1_04_wctlmt_1__18__04_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wctlmt_1__18__05_INT_M':hp.qloguniform('_1_04_wctlmt_1__18__05_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_04_wctlmt_1__18__06_A':hp.choice('_1_04_wctlmt_1__18__06_A', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wctlmt_1__18__07_auto_param_401':hp.choice('_1_04_wctlmt_1__18__07_auto_param_401', [ + { + '_1_04_wctlmt_1__18__07__00__00_W_HIDDEN':'0', + '_1_04_wctlmt_1__18__07__00__01_1_W':hp.choice('_1_04_wctlmt_1__18__07__00__01_1_W', ['0', ]), + }, + { + '_1_04_wctlmt_1__18__07__01__00_W_HIDDEN':'1', + '_1_04_wctlmt_1__18__07__01__01_2_W':hp.uniform('_1_04_wctlmt_1__18__07__01__01_2_W', 0.0, 1.0), + }, + ]), + '_1_04_wctlmt_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wctrept_0_QUOTE_START_B':'weka.classifiers.trees.REPTree', + '_1_04_wctrept_1__19__01_INT_M':hp.qloguniform('_1_04_wctrept_1__19__01_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_04_wctrept_1__19__02_V':hp.loguniform('_1_04_wctrept_1__19__02_V', math.log(1.0E-5), math.log(0.1)), + '_1_04_wctrept_1__19__03_P':hp.choice('_1_04_wctrept_1__19__03_P', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wctrept_1__19__04_auto_param_405':hp.choice('_1_04_wctrept_1__19__04_auto_param_405', [ + { + '_1_04_wctrept_1__19__04__00__00_depth_HIDDEN':'0', + '_1_04_wctrept_1__19__04__00__01_1_INT_L':hp.choice('_1_04_wctrept_1__19__04__00__01_1_INT_L', ['-1', ]), + }, + { + '_1_04_wctrept_1__19__04__01__00_depth_HIDDEN':'1', + '_1_04_wctrept_1__19__04__01__01_2_INT_L':hp.quniform('_1_04_wctrept_1__19__04__01__01_2_INT_L', 2.0, 20.0,1.0), + }, + ]), + '_1_04_wctrept_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wctrf_0_QUOTE_START_B':'weka.classifiers.trees.RandomForest', + '_1_04_wctrf_1__20__01_INT_I':hp.qloguniform('_1_04_wctrf_1__20__01_INT_I', math.log(2.0), math.log(256.0), 1.0), + '_1_04_wctrf_1__20__02_auto_param_409':hp.choice('_1_04_wctrf_1__20__02_auto_param_409', [ + { + '_1_04_wctrf_1__20__02__00__00_features_HIDDEN':'0', + '_1_04_wctrf_1__20__02__00__01_1_INT_K':hp.choice('_1_04_wctrf_1__20__02__00__01_1_INT_K', ['1', ]), + }, + { + '_1_04_wctrf_1__20__02__01__00_features_HIDDEN':'1', + '_1_04_wctrf_1__20__02__01__01_2_INT_K':hp.qloguniform('_1_04_wctrf_1__20__02__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0), + }, + ]), + '_1_04_wctrf_1__20__03_auto_param_412':hp.choice('_1_04_wctrf_1__20__03_auto_param_412', [ + { + '_1_04_wctrf_1__20__03__00__00_depth_HIDDEN':'0', + '_1_04_wctrf_1__20__03__00__01_1_INT_depth':hp.choice('_1_04_wctrf_1__20__03__00__01_1_INT_depth', ['1', ]), + }, + { + '_1_04_wctrf_1__20__03__01__00_depth_HIDDEN':'1', + '_1_04_wctrf_1__20__03__01__01_2_INT_depth':hp.quniform('_1_04_wctrf_1__20__03__01__01_2_INT_depth', 2.0, 20.0,1.0), + }, + ]), + '_1_04_wctrf_2_QUOTE_END':'REMOVED', + }, + { + '_1_04_wctrt_0_QUOTE_START_B':'weka.classifiers.trees.RandomTree', + '_1_04_wctrt_1__21__01_INT_M':hp.qloguniform('_1_04_wctrt_1__21__01_INT_M', math.log(1.0), math.log(64.0), 1.0), + '_1_04_wctrt_1__21__02_U':hp.choice('_1_04_wctrt_1__21__02_U', ['REMOVE_PREV', 'REMOVED', ]), + '_1_04_wctrt_1__21__03_auto_param_416':hp.choice('_1_04_wctrt_1__21__03_auto_param_416', [ + { + '_1_04_wctrt_1__21__03__00__00_features_HIDDEN':'0', + '_1_04_wctrt_1__21__03__00__01_1_INT_K':hp.choice('_1_04_wctrt_1__21__03__00__01_1_INT_K', ['0', ]), + }, + { + '_1_04_wctrt_1__21__03__01__00_features_HIDDEN':'1', + '_1_04_wctrt_1__21__03__01__01_2_INT_K':hp.qloguniform('_1_04_wctrt_1__21__03__01__01_2_INT_K', math.log(2.0), math.log(32.0), 1.0), + }, + ]), + '_1_04_wctrt_1__21__04_auto_param_419':hp.choice('_1_04_wctrt_1__21__04_auto_param_419', [ + { + '_1_04_wctrt_1__21__04__00__00_depth_HIDDEN':'0', + '_1_04_wctrt_1__21__04__00__01_1_INT_depth':hp.choice('_1_04_wctrt_1__21__04__00__01_1_INT_depth', ['0', ]), + }, + { + '_1_04_wctrt_1__21__04__01__00_depth_HIDDEN':'1', + '_1_04_wctrt_1__21__04__01__01_2_INT_depth':hp.quniform('_1_04_wctrt_1__21__04__01__01_2_INT_depth', 2.0, 20.0,1.0), + }, + ]), + '_1_04_wctrt_1__21__05_auto_param_422':hp.choice('_1_04_wctrt_1__21__05_auto_param_422', [ + { + '_1_04_wctrt_1__21__05__00__00_back_HIDDEN':'0', + '_1_04_wctrt_1__21__05__00__01_1_INT_N':hp.choice('_1_04_wctrt_1__21__05__00__01_1_INT_N', ['0', ]), + }, + { + '_1_04_wctrt_1__21__05__01__00_back_HIDDEN':'1', + '_1_04_wctrt_1__21__05__01__01_2_INT_N':hp.quniform('_1_04_wctrt_1__21__05__01__01_2_INT_N', 2.0, 5.0,1.0), + }, + ]), + '_1_04_wctrt_2_QUOTE_END':'REMOVED', + }, + ]), + ), + ]), + ), + ]), + ), + ]), + ), + ]), + ), + ), + ]), + ]), +) diff --git a/tests/search_spaces/nips2011.py b/tests/search_spaces/nips2011.py new file mode 100644 index 00000000..80499bcb --- /dev/null +++ b/tests/search_spaces/nips2011.py @@ -0,0 +1,56 @@ +""" +Neural Network (NNet) and Deep Belief Network (DBN) search spaces used in [1] +and [2]. + +The functions in this file return pyll graphs that can be used as the `space` +argument to e.g. `hyperopt.fmin`. The pyll graphs include hyperparameter +constructs (e.g. `hyperopt.hp.uniform`) so `hyperopt.fmin` can perform +hyperparameter optimization. + +See ./skdata_learning_algo.py for example usage of these functions. + + +[1] Bergstra, J., Bardenet, R., Bengio, Y., Kegl, B. (2011). Algorithms +for Hyper-parameter optimization, NIPS 2011. + +[2] Bergstra, J., Bengio, Y. (2012). Random Search for Hyper-Parameter +Optimization, JMLR 13:281--305. + +""" + +""" +CHANGED TO WORK AS SEARCHSPACE IN THE BBoM Framework +""" + +__author__ = "James Bergstra" +__license__ = "BSD-3" + +import numpy as np + +from hyperopt import hp + +space = {'preproc': hp.choice('preproc', [{ + 'preproc' : 0}, { + 'preproc' : 1, # Column Normalization + 'colnorm_thresh' : hp.loguniform('colnorm_thresh', np.log(1e-9), np.log(1e-3)), + }, { + 'preproc' : 2, # PCA + 'pca_energy' : hp.uniform('pca_energy', .5, 1), + }]), + 'nhid1': hp.qloguniform('nhid1', np.log(16), np.log(1024), q=16), + 'dist1': hp.choice('dist1', [0, 1]), # 0 = Uniform, 1 = Normal + 'scale_heur1': hp.choice('scale_heur1', [{ + 'scale_heur1' : 0, # Old + 'scale_mult1': hp.uniform('scale_mult1', .2, 2)}, { + 'scale_heur1': 1}]), # Glorot + 'squash' : hp.choice('squash', [0, 1]), # ['tanh', 'logistic'] + 'iseed': hp.choice('iseed', [0, 1, 2, 3]), # Are 5, 6, 7, 8 + 'batch_size': hp.choice('batch_size', [0, 1]), # 20 or 100 + 'lr': hp.lognormal('lr', np.log(.01), 3.), + 'lr_anneal_start': hp.qloguniform('lr_anneal_start', np.log(100), np.log(10000), q=1), + 'l2_penalty': hp.choice('l2_penalty', [{ + 'l2_penalty' : 0}, { # Zero + 'l2_penalty' : 1, # notzero + 'l2_penalty_nz': hp.lognormal('l2_penalty_nz', np.log(1.0e-6), 2.)}]) + } + diff --git a/tests/test_benchmark_util.py b/tests/test_benchmark_util.py new file mode 100644 index 00000000..ca1d310a --- /dev/null +++ b/tests/test_benchmark_util.py @@ -0,0 +1,68 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import unittest +import sys + +import HPOlib.benchmark_util as benchmark_util + + +class BenchmarkUtilTest(unittest.TestCase): + def setUp(self): + # Change into the parent of the test directory + os.chdir(os.path.join("..", os.path.dirname(os.path.realpath(__file__)))) + + # Make sure there is no config file + try: + os.remove("./config.cfg") + except: + pass + + def test_read_parameters_from_command_line(self): + # Legal call + sys.argv = ["test.py", "--folds", "10", "--fold", "0", "--params", "-x", + "3"] + args, params = benchmark_util.parse_cli() + self.assertEqual(params, {'x': '3'}) + self.assertEqual(args, {'folds': '10', 'fold': '0'}) + + # illegal call, arguments with one minus before --params + sys.argv = ["test.py", "-folds", "10", "--fold", "0", "--params", "-x", + "3"] + with self.assertRaises(ValueError) as cm1: + benchmark_util.parse_cli() + self.assertEqual(cm1.exception.message, "You either try to use arguments" + " with only one leading minus or try to specify a " + "hyperparameter before the --params argument. test.py" + " -folds 10 --fold 0 --params -x 3") + + # illegal call, trying to specify an arguments after --params + sys.argv = ["test.py", "--folds", "10", "--params", "-x", + "'3'", "--fold", "0"] + with self.assertRaises(ValueError) as cm5: + benchmark_util.parse_cli() + self.assertEqual(cm5.exception.message, "You are trying to specify an argument after the " + "--params argument. Please change the order.") + + # illegal call, no - in front of parameter name + sys.argv = ["test_cv.py", "--params", "x", "'5'"] + with self.assertRaises(ValueError) as cm2: + benchmark_util.parse_cli() + self.assertEqual(cm2.exception.message, "Illegal command line string, expected a hyperpara" + "meter starting with - but found x") \ No newline at end of file diff --git a/tests/test_cv.py b/tests/test_cv.py new file mode 100644 index 00000000..a05db27f --- /dev/null +++ b/tests/test_cv.py @@ -0,0 +1,125 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import unittest +import numpy as np +import sys + +import HPOlib.cv as cv +try: + import hyperopt +except: + # TODO: Remove this Hackiness when installation fully works! + import HPOlib + hyperopt_path = os.path.join(os.path.dirname(os.path.abspath( + HPOlib.__file__)), "optimizers/hyperopt_august2013_mod") + print hyperopt_path + sys.path.append(hyperopt_path) + import hyperopt + + +class CVTest(unittest.TestCase): + def setUp(self): + # Change into the parent of the test directory + os.chdir(os.path.join("..", os.path.dirname(os.path.realpath(__file__)))) + + # Make sure there is no config file + try: + os.remove("./config.cfg") + except: + pass + + def test_read_parameters_from_command_line(self): + # Legal call + sys.argv = ["test_cv.py", "-x", "'5'", "-name", "'Koenigsberghausen'", + "-y", "'5.0'", "-z", "'-3.0'"] + params = cv.read_params_from_command_line() + self.assertEqual(params, {'x': '5', 'name': 'Koenigsberghausen', + 'y': '5.0', 'z': '-3.0'}) + + # illegal call, no - in front of parameter name + sys.argv = ["test_cv.py", "x", "'5'"] + self.assertRaises(ValueError, cv.read_params_from_command_line) + + # illegal call, no single quotation mark around second parameter + sys.argv = ["test_cv.py", "-x", "5"] + self.assertRaises(ValueError, cv.read_params_from_command_line) + + # illegal call, no parameter value + sys.argv = ["test_cv.py", "-x", "-y"] + self.assertRaises(ValueError, cv.read_params_from_command_line) + + def test_parameter_flattening(self): + def naive_old_implementation(params): + _params_to_check = list(params.keys()) + _new_dict = dict() + while len(_params_to_check) != 0: + p = _params_to_check.pop() + if isinstance(params[p], dict): + _params_to_check.extend(params[p].keys()) + params.update(params[p]) + elif isinstance(params[p], np.ndarray) or \ + isinstance(params[p], list): + _new_dict[p] = params[p][0] + else: + _new_dict[p] = params[p] + return _new_dict + + # Branin + import HPOlib.benchmarks.branin.tpe.space + space = HPOlib.benchmarks.branin.tpe.space.space + + for i in range(100): + sample = hyperopt.pyll.stochastic.sample(space) + flatten = cv.flatten_parameter_dict(sample) + flatten_old = naive_old_implementation(sample) + self.assertEqual(len(flatten), 2) + self.assertEqual(type(flatten), dict) + self.assertEqual(type(flatten["x"]), float) + self.assertEqual(type(flatten["y"]), float) + self.assertEqual(flatten, flatten_old) + + # HPNnet + import tests.search_spaces.nips2011 + space = tests.search_spaces.nips2011.space + + for i in range(100): + sample = hyperopt.pyll.stochastic.sample(space) + flatten = cv.flatten_parameter_dict(sample) + flatten_old = naive_old_implementation(sample) + # print flatten + # print flatten_old + self.assertLessEqual(len(flatten), 13) + self.assertGreaterEqual(len(flatten), 10) + self.assertEqual(type(flatten), dict) + self.assertEqual(flatten, flatten_old) + + # AutoWEKA + import tests.search_spaces.autoweka + space = tests.search_spaces.autoweka.space + for i in range(100): + sample = hyperopt.pyll.stochastic.sample(space) + flatten = cv.flatten_parameter_dict(sample) + self.assertIn("attributesearch", flatten.keys()) + self.assertIn("targetclass", flatten.keys()) + # print flatten + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_data_utils.py b/tests/test_data_utils.py new file mode 100644 index 00000000..61831676 --- /dev/null +++ b/tests/test_data_utils.py @@ -0,0 +1,105 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cPickle +import ConfigParser +from contextlib import contextmanager +import numpy as np +import os +import sys +import unittest + +import HPOlib.data_util as data_util + +@contextmanager +def redirected(out=sys.stdout, err=sys.stderr): + saved = sys.stdout, sys.stderr + sys.stdout, sys.stderr = out, err + try: + yield + finally: + sys.stdout, sys.stderr = saved + + +class DataUtilTest(unittest.TestCase): + def setUp(self): + # Change into the test directory + os.chdir(os.path.dirname(os.path.realpath(__file__))) + + # Make sure there is no config file + try: + os.remove("./config.cfg") + except os.error: + pass + + def tearDown(self): + try: + os.remove("./test_get_trial_index.pkl") + except OSError: + pass + + def test_load_file(self): + # Test numpy arrays + train_data = np.zeros((100, 100)) + np.save("train_data.npy", train_data) + data = data_util.load_file("train_data.npy", "numpy", 100) + self.assertTrue((train_data == data).all()) + data = data_util.load_file("train_data.npy", "numpy", 10) + self.assertTrue((train_data[:10] == data).all()) + os.remove("train_data.npy") + + # Test pickle files + train_data = np.zeros((100, 100)) + fh = open("train_data.pkl", "w") + cPickle.dump(train_data, fh) + fh.close() + data = data_util.load_file("train_data.pkl", "pickle", 100) + self.assertTrue((train_data == data).all()) + data = data_util.load_file("train_data.pkl", "pickle", 10) + self.assertTrue((train_data[:10] == data).all()) + os.remove("train_data.pkl") + + # Test zipped pickle files + train_data = np.zeros((100, 100)) + from gzip import GzipFile as gfile + fh = gfile("train_data.pkl.gz", "w") + cPickle.dump(train_data, fh) + fh.close() + data = data_util.load_file("train_data.pkl.gz", "gfile", 100) + self.assertTrue((train_data == data).all()) + data = data_util.load_file("train_data.pkl.gz", "gfile", 10) + self.assertTrue((train_data[:10] == data).all()) + os.remove("train_data.pkl.gz") + + # Test wrong data file type + self.assertRaises(IOError, data_util.load_file, + "test.sh", "uditare", 1) + self.assertRaises(IOError, data_util.load_file, + "", "uditare", 1) + + @unittest.skip("Not implemented yet") + def test_custom_split(self): + self.fail() + + @unittest.skip("Not implemented yet") + def test_prepare_cv_for_fold(self): + self.fail() + + @unittest.skip("Not implemented yet") + def test_remove_param_metadata(self): + self.fail() \ No newline at end of file diff --git a/tests/test_experiment.py b/tests/test_experiment.py new file mode 100644 index 00000000..dfc99c3d --- /dev/null +++ b/tests/test_experiment.py @@ -0,0 +1,237 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import numpy as np +import unittest + +import HPOlib.Experiment as Experiment + + +def _sanity_check(experiment): + experiment._sanity_check() + pass + + +class ExperimentTest(unittest.TestCase): + # TODO: Test timing mechanism!!! + def setUp(self): + # Change into the test directory + os.chdir(os.path.dirname(os.path.realpath(__file__))) + + try: + os.remove("test_exp.pkl") + except OSError: + pass + + def tearDown(self): + # Delete the test experiment + try: + os.remove("test_exp.pkl") + except OSError: + pass + try: + os.remove("test_exp.pkl.lock") + except OSError: + pass + + def test_init(self): + # TODO: Somehow test in which folder the experiment is created + # TODO: Remove the case that it is always saved automatically + exp = Experiment.Experiment(".", "test_exp") + _sanity_check(exp) + exp.title = "test" + del exp + + # Make sure reloading works + exp = Experiment.Experiment(".", "test_exp") + self.assertEqual(exp.title, "test") + del exp + #self.fail() + + def test_status_getters(self): + experiment = Experiment.Experiment(".", "test_exp", folds=2) + # Candidate jobs + experiment.add_job({"x": "0"}) + experiment.add_job({"x": "1"}) + # Complete jobs + experiment.add_job({"x": "2"}) + experiment.set_one_fold_running(2, 0) + experiment.set_one_fold_complete(2, 0, 1, 1) + experiment.set_one_fold_running(2, 1) + experiment.set_one_fold_complete(2, 1, 1, 1) + experiment.add_job({"x": "3"}) + experiment.set_one_fold_running(3, 0) + experiment.set_one_fold_complete(3, 0, 1, 1) + experiment.set_one_fold_running(3, 1) + experiment.set_one_fold_complete(3, 1, 1, 1) + # Incomplete jobs + experiment.add_job({"x": "4"}) + experiment.set_one_fold_running(4, 0) + experiment.set_one_fold_complete(4, 0, 1, 1) + experiment.add_job({"x": "5"}) + experiment.set_one_fold_running(5, 0) + experiment.set_one_fold_complete(5, 0, 1, 1) + # Running Jobs + experiment.add_job({"x": "6"}) + experiment.set_one_fold_running(6, 0) + experiment.add_job({"x": "7"}) + experiment.set_one_fold_running(7, 0) + # Broken Jobs + experiment.add_job({"x": "8"}) + experiment.set_one_fold_running(8, 0) + experiment.set_one_fold_crashed(8, 0, 1000, 1) + experiment.add_job({"x": "9"}) + experiment.set_one_fold_running(9, 0) + experiment.set_one_fold_crashed(9, 0, 1000, 1) + experiment.set_one_fold_running(9, 1) + experiment.set_one_fold_crashed(9, 1, 1000, 1) + self.assertEqual(len(experiment.get_candidate_jobs()), 2) + self.assertEqual(len(experiment.get_complete_jobs()), 2) + self.assertEqual(len(experiment.get_incomplete_jobs()), 3) + self.assertEqual(len(experiment.get_running_jobs()), 2) + self.assertEqual(len(experiment.get_broken_jobs()), 1) + self.assertEqual(experiment.trials[9]['result'], 1000) + self.assertNotEqual(experiment.trials[8]['result'], np.NaN) + + def test_get_arg_best(self): + experiment = Experiment.Experiment(".", "test_exp", folds=2) + [experiment.add_job({"x": i}) for i in range(10)] + [experiment.set_one_fold_running(i, 0) for i in range(10)] + [experiment.set_one_fold_complete(i, 0, 10 - i, 1) for i in range(10)] + [experiment.set_one_fold_running(i, 1) for i in range(10)] + [experiment.set_one_fold_complete(i, 1, i, 1) for i in range(10)] + self.assertEqual(experiment.get_arg_best(), 0) + + def test_get_arg_best_NaNs(self): + experiment = Experiment.Experiment(".", "test_exp", folds=2) + [experiment.add_job({"x": i}) for i in range(10)] + [experiment.set_one_fold_running(i, 0) for i in range(10)] + [experiment.set_one_fold_complete(i, 0, 10 - i, 1) for i in range(10)] + [experiment.set_one_fold_running(i, 1) for i in range(10)] + [experiment.set_one_fold_complete(i, 1, i, 1) for i in range(1, 10)] + self.assertEqual(experiment.get_arg_best(), 1) + + def test_add_job(self): + exp = Experiment.Experiment(".", "test_exp", folds=5) + self.assertEqual(len(exp.trials), 0) + self.assertEqual(len(exp.instance_order), 0) + + _id = exp.add_job({"x": 1, "y": 2}) + self.assertEqual(len(exp.trials), 1) + self.assertEqual(len(exp.instance_order), 0) + self.assertEqual(exp.get_trial_from_id(_id)['instance_results'].shape, + (5,)) + self.assertEqual(exp.get_trial_from_id(_id)['instance_durations'].shape, + (5,)) + self.assertEqual(exp.get_trial_from_id(_id)['instance_status'].shape, + (5,)) + self.assertDictEqual(exp.get_trial_from_id(_id)['params'], {"x": 1, "y": 2}) + _sanity_check(exp) + + def test_set_one_fold_crashed(self): + experiment = Experiment.Experiment(".", "test_exp", folds=1) + experiment.add_job({"x": 0}) + experiment.set_one_fold_running(0, 0) + experiment.set_one_fold_crashed(0, 0, 1000, 0) + + def test_one_fold_workflow(self): + experiment = Experiment.Experiment(".", "test_exp", folds=5) + trial_index = experiment.add_job({"x": 5}) + experiment.set_one_fold_running(trial_index, 0) + self.assertEqual(len(experiment.get_broken_jobs()), 0) + self.assertEqual(len(experiment.get_complete_jobs()), 0) + self.assertEqual(len(experiment.get_running_jobs()), 1) + self.assertEqual(experiment.get_trial_from_id(trial_index) + ['instance_status'][0], Experiment.RUNNING_STATE) + + experiment.set_one_fold_complete(trial_index, 0, 1, 1) + self.assertEqual(len(experiment.get_complete_jobs()), 0) + self.assertEqual(len(experiment.get_incomplete_jobs()), 1) + self.assertEqual(experiment.get_trial_from_id(trial_index) + ['instance_status'][0], Experiment.COMPLETE_STATE) + + experiment.set_one_fold_running(trial_index, 1) + experiment.set_one_fold_complete(trial_index, 1, 2, 1) + self.assertEqual(len(experiment.get_incomplete_jobs()), 1) + experiment.set_one_fold_running(trial_index, 2) + experiment.set_one_fold_complete(trial_index, 2, 3, 1) + self.assertEqual(len(experiment.get_incomplete_jobs()), 1) + experiment.set_one_fold_running(trial_index, 3) + experiment.set_one_fold_complete(trial_index, 3, 4, 1) + self.assertEqual(len(experiment.get_incomplete_jobs()), 1) + experiment.set_one_fold_running(trial_index, 4) + experiment.set_one_fold_complete(trial_index, 4, 5, 1) + self.assertEqual(len(experiment.trials), 1) + self.assertTrue((experiment.get_trial_from_id(trial_index)["instance_results"] == [1, 2, 3, 4, 5]).all()) + self.assertEqual(len(experiment.get_complete_jobs()), 1) + self.assertEqual(experiment.get_trial_from_id(trial_index)['status'], + Experiment.COMPLETE_STATE) + + trial_index1 = experiment.add_job({"x": 6}) + self.assertEqual(len(experiment.get_complete_jobs()), 1) + self.assertEqual(len(experiment.get_candidate_jobs()), 1) + experiment.set_one_fold_running(trial_index1, 3) + self.assertTrue((experiment.get_trial_from_id(trial_index1) + ['instance_status'] == [0, 0, 0, 2, 0]).all()) + experiment.set_one_fold_complete(trial_index1, 3, 1, 1) + self.assertTrue((experiment.get_trial_from_id(trial_index1) + ['instance_status'] == [0, 0, 0, 3, 0]).all()) + + self.assertEqual(experiment.instance_order, [(0, 0), (0, 1), (0, 2), + (0, 3), (0, 4), (1, 3)]) + self.assertEqual(experiment.total_wallclock_time, 6) + self.assertTrue((experiment.get_trial_from_id(trial_index) + ["instance_durations"] == [1, 1, 1, 1, 1]).all()) + + # Check that check_cv_finished kicked in + self.assertEqual(experiment.get_trial_from_id(trial_index) + ["result"], 3.0) + self.assertAlmostEqual(experiment.get_trial_from_id(trial_index)['std'], 1.4142135623730951) + # Check that check_cv_finished did not kick in + self.assertNotEqual(experiment.get_trial_from_id(trial_index1) + ["result"], experiment.get_trial_from_id(trial_index1) + ["result"]) + self.assertNotEqual(experiment.get_trial_from_id(trial_index1)['std'], + experiment.get_trial_from_id(trial_index1)['std']) + self.assertEqual(len(experiment.trials), 2) + _sanity_check(experiment) + + def test_remove_all_but_first_runs(self): + experiment = Experiment.Experiment(".", "test_exp", folds=5) + for i in range(5): + experiment.add_job({"x": i}) + experiment.set_one_fold_running(i, i) + experiment.set_one_fold_complete(i, i, 1, 1) + experiment.set_one_fold_running(2, 3) + experiment.set_one_fold_complete(2, 3, 1, 1) + + self.assertEqual(len(experiment.get_incomplete_jobs()), 5) + self.assertEqual(len(experiment.instance_order), 6) + + experiment.remove_all_but_first_runs(3) + self.assertEqual(len(experiment.get_incomplete_jobs()), 3) + self.assertEqual(len(experiment.instance_order), 3) + self.assertTrue((experiment.get_trial_from_id(2)["instance_status"] == + [0, 0, 3, 0, 0]).all()) + + _sanity_check(experiment) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_get_trial_index.pkl b/tests/test_get_trial_index.pkl new file mode 100644 index 00000000..98aaf140 --- /dev/null +++ b/tests/test_get_trial_index.pkl @@ -0,0 +1,421 @@ +(dp1 +S'cv_endtime' +p2 +(lp3 +sS'folds' +p4 +I1 +sS'cv_starttime' +p5 +(lp6 +sS'endtime' +p7 +(lp8 +sS'optimizer' +p9 +NsS'optimizer_time' +p10 +(lp11 +sS'instance_order' +p12 +(lp13 +(I0 +I0 +tp14 +a(I1 +I0 +tp15 +a(I2 +I0 +tp16 +a(I3 +I0 +tp17 +a(I4 +I0 +tp18 +asS'max_wallclock_time' +p19 +F1.7976931348623157e+308 +sS'title' +p20 +NsS'trials' +p21 +(lp22 +(dp23 +S'status' +p24 +I3 +sS'std' +p25 +cnumpy.core.multiarray +scalar +p26 +(cnumpy +dtype +p27 +(S'f8' +I0 +I1 +tRp28 +(I3 +S'<' +NNNI-1 +I-1 +I0 +tbS'\x00\x00\x00\x00\x00\x00\x00\x00' +tRp29 +sS'instance_durations' +p30 +cnumpy.core.multiarray +_reconstruct +p31 +(cnumpy +ndarray +p32 +(I0 +tS'b' +tRp33 +(I1 +(I1 +tg28 +I00 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tbsS'instance_results' +p34 +g31 +(g32 +(I0 +tS'b' +tRp35 +(I1 +(I1 +tg28 +I00 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tbsS'test_error' +p36 +Fnan +sS'params' +p37 +(dp38 +S'x' +S'1' +ssS'result' +p39 +g26 +(g28 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tRp40 +sS'duration' +p41 +g26 +(g28 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tRp42 +sS'instance_status' +p43 +g31 +(g32 +(I0 +tS'b' +tRp44 +(I1 +(I1 +tg27 +(S'i4' +I0 +I1 +tRp45 +(I3 +S'<' +NNNI-1 +I-1 +I0 +tbI00 +S'\x03\x00\x00\x00' +tbsa(dp46 +g24 +I3 +sg25 +g26 +(g28 +S'\x00\x00\x00\x00\x00\x00\x00\x00' +tRp47 +sg30 +g31 +(g32 +(I0 +tS'b' +tRp48 +(I1 +(I1 +tg28 +I00 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tbsg34 +g31 +(g32 +(I0 +tS'b' +tRp49 +(I1 +(I1 +tg28 +I00 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tbsg36 +Fnan +sg37 +(dp50 +S'x' +S'2' +ssg39 +g26 +(g28 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tRp51 +sg41 +g26 +(g28 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tRp52 +sg43 +g31 +(g32 +(I0 +tS'b' +tRp53 +(I1 +(I1 +tg45 +I00 +S'\x03\x00\x00\x00' +tbsa(dp54 +g24 +I3 +sg25 +g26 +(g28 +S'\x00\x00\x00\x00\x00\x00\x00\x00' +tRp55 +sg30 +g31 +(g32 +(I0 +tS'b' +tRp56 +(I1 +(I1 +tg28 +I00 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tbsg34 +g31 +(g32 +(I0 +tS'b' +tRp57 +(I1 +(I1 +tg28 +I00 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tbsg36 +Fnan +sg37 +(dp58 +S'x' +S'3' +ssg39 +g26 +(g28 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tRp59 +sg41 +g26 +(g28 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tRp60 +sg43 +g31 +(g32 +(I0 +tS'b' +tRp61 +(I1 +(I1 +tg45 +I00 +S'\x03\x00\x00\x00' +tbsa(dp62 +g24 +I3 +sg25 +g26 +(g28 +S'\x00\x00\x00\x00\x00\x00\x00\x00' +tRp63 +sg30 +g31 +(g32 +(I0 +tS'b' +tRp64 +(I1 +(I1 +tg28 +I00 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tbsg34 +g31 +(g32 +(I0 +tS'b' +tRp65 +(I1 +(I1 +tg28 +I00 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tbsg36 +Fnan +sg37 +(dp66 +S'x' +S'4' +ssg39 +g26 +(g28 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tRp67 +sg41 +g26 +(g28 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tRp68 +sg43 +g31 +(g32 +(I0 +tS'b' +tRp69 +(I1 +(I1 +tg45 +I00 +S'\x03\x00\x00\x00' +tbsa(dp70 +g24 +I3 +sg25 +g26 +(g28 +S'\x00\x00\x00\x00\x00\x00\x00\x00' +tRp71 +sg30 +g31 +(g32 +(I0 +tS'b' +tRp72 +(I1 +(I1 +tg28 +I00 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tbsg34 +g31 +(g32 +(I0 +tS'b' +tRp73 +(I1 +(I1 +tg28 +I00 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tbsg36 +Fnan +sg37 +(dp74 +S'x' +S'5' +ssg39 +g26 +(g28 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tRp75 +sg41 +g26 +(g28 +S'\x00\x00\x00\x00\x00\x00\xf0?' +tRp76 +sg43 +g31 +(g32 +(I0 +tS'b' +tRp77 +(I1 +(I1 +tg45 +I00 +S'\x03\x00\x00\x00' +tbsa(dp78 +g24 +I0 +sg25 +Fnan +sg30 +g31 +(g32 +(I0 +tS'b' +tRp79 +(I1 +(I1 +tg28 +I00 +S'\x00\x00\x00\x00\x00\x00\xf8\x7f' +tbsg34 +g31 +(g32 +(I0 +tS'b' +tRp80 +(I1 +(I1 +tg28 +I00 +S'\x00\x00\x00\x00\x00\x00\xf8\x7f' +tbsg36 +Fnan +sg37 +g58 +sg39 +Fnan +sg41 +Fnan +sg43 +g31 +(g32 +(I0 +tS'b' +tRp81 +(I1 +(I1 +tg45 +I00 +S'\x00\x00\x00\x00' +tbsasS'total_wallclock_time' +p82 +I5 +sS'experiment_name' +p83 +S'test_get_trial_index' +p84 +sS'starttime' +p85 +(lp86 +s. \ No newline at end of file diff --git a/tests/test_gridsearch.py b/tests/test_gridsearch.py new file mode 100644 index 00000000..e9e3974b --- /dev/null +++ b/tests/test_gridsearch.py @@ -0,0 +1,67 @@ +from collections import OrderedDict +import unittest +import os + +import HPOlib.optimizers.gridsearch.gridsearch as gridsearch + + +class GridSearchTest(unittest.TestCase): + def setUp(self): + self.hyperparameters = OrderedDict() + self.hyperparameters["x"] = [-5, 0, 5, 10] + self.hyperparameters["y"] = [0, 5, 10, 15] + + def test_parse_hyperopt_string(self): + hyperparameter_string = "x {-5, 0, 5, 10}\ny {0, 5, 10, 15}" + expected = OrderedDict([["x", ["-5", "0", "5", "10"]], + ["y", ["0", "5", "10", "15"]]]) + ret = gridsearch.parse_hyperparameter_string(hyperparameter_string) + self.assertEqual(ret, expected) + + hyperparameter_string = "x {-5, 0, 5, 10} [5]\ny {0, 5, 10, 15}" + ret = gridsearch.parse_hyperparameter_string(hyperparameter_string) + self.assertEqual(ret, expected) + + hyperparameter_string = "x {-5, 0, 5, 10}\ny {0, 5, 10, 15} [5]" + ret = gridsearch.parse_hyperparameter_string(hyperparameter_string) + self.assertEqual(ret, expected) + + hyperparameter_string = "x {-5, 0, 5, 10}\ny 0, 5, 10, 15} [5]" + self.assertRaises(ValueError, gridsearch.parse_hyperparameter_string, + hyperparameter_string) + + def test_construct_cli_call(self): + cli_call = gridsearch.construct_cli_call("cv.py", {"x": -5, "y": 0}) + self.assertEqual(cli_call, "cv.py -x \"'-5'\" -y \"'0'\"") + + + def test_build_grid(self): + grid = gridsearch.build_grid(self.hyperparameters) + self.assertIsInstance(grid, list) + self.assertEqual(len(grid), 16) + self.assertDictEqual(grid[0], {"x": -5, "y": 0}) + self.assertDictEqual(grid[3], {"x": -5, "y": 15}) + self.assertDictEqual(grid[9], {"x": 5, "y": 5}) + self.assertDictEqual(grid[15], {"x": 10, "y": 15}) + + def test_build_grid_3d(self): + self.hyperparameters["z"] = [5, 10, 15, 20] + grid = gridsearch.build_grid(self.hyperparameters) + self.assertIsInstance(grid, list) + self.assertEqual(len(grid), 64) + self.assertDictEqual(grid[0], {"x": -5, "y": 0, "z": 5}) + self.assertDictEqual(grid[16], {"x": 0, "y": 0, "z": 5}) + + def test_perform_grid_search(self): + from numpy import pi, cos + + def branin(params): + x = params["x"] + y = params["y"] + result = (y-(5.1/(4*pi**2))*x**2+5*x/pi-6)**2 + result += 10*(1-1/(8*pi))*cos(x)+10 + return result + + grid = gridsearch.build_grid(self.hyperparameters) + gridsearch.perform_gridsearch(branin, grid) + diff --git a/tests/test_runsolver_wrapper.py b/tests/test_runsolver_wrapper.py new file mode 100644 index 00000000..0b3a0ed8 --- /dev/null +++ b/tests/test_runsolver_wrapper.py @@ -0,0 +1,234 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import ConfigParser +import os +import sys +import unittest + +import HPOlib.Experiment as Experiment +import HPOlib.runsolver_wrapper as runsolver_wrapper + + +class RunsolverWrapperTest(unittest.TestCase): + def setUp(self): + # Change into the test directory + os.chdir(os.path.dirname(os.path.realpath(__file__))) + + # Make sure there is no config file + try: + os.remove("config.cfg") + except OSError: + pass + try: + os.remove("./tests.pkl") + except OSError: + pass + try: + os.remove("./tests.pkl.lock") + except OSError: + pass + + def tearDown(self): + try: + os.remove("config.cfg") + except OSError: + pass + try: + os.remove("./tests.pkl") + except OSError: + pass + try: + os.remove("./tests.pkl") + except OSError: + pass + + def test_read_runsolver_output(self): + cpu_time, wallclock_time, error = runsolver_wrapper\ + .read_runsolver_output("runsolver_positive.txt") + self.assertAlmostEqual(cpu_time, 0.188011) + self.assertAlmostEqual(wallclock_time, 0.259524) + self.assertTrue(error is None) + + def test_read_runsolver_output_wallclock(self): + cpu_time, wallclock_time, error = runsolver_wrapper\ + .read_runsolver_output("runsolver_wallclock_time_limit.txt") + self.assertAlmostEqual(cpu_time, 0.044002) + self.assertAlmostEqual(wallclock_time, 0.066825) + self.assertEqual(error, "Wall clock time exceeded") + + def test_read_runsolver_output_vsize(self): + cpu_time, wallclock_time, error = runsolver_wrapper\ + .read_runsolver_output("runsolver_vsize_exceeded.txt") + self.assertAlmostEqual(cpu_time, 0.016) + self.assertAlmostEqual(wallclock_time,0.039276) + self.assertEqual(error, "VSize exceeded") + + def test_read_runsolver_output_warning(self): + cpu_time, wallclock_time, error = runsolver_wrapper\ + .read_runsolver_output("runsolver_positive_with_warning.txt") + self.assertAlmostEqual(cpu_time, 0.820027) + self.assertAlmostEqual(wallclock_time, 1.00203) + self.assertTrue(error is None) + + def test_read_run_instance_output_no_result(self): + result_array, result_string = runsolver_wrapper.\ + read_run_instance_output("run_instance_no_result.txt") + self.assertTrue(result_string is None) + self.assertTrue(result_array is None) + + def test_read_run_instance_output_result(self): + result_array, result_string = runsolver_wrapper.\ + read_run_instance_output("run_instance_result.txt") + self.assertEqual(result_string, "Result for ParamILS: SAT, 0.35, 1, " + "0.5, -1, Random file") + self.assertListEqual(result_array, ["Result", "for", "ParamILS:", + "SAT", "0.35", "1", "0.5", + "-1", "Random", "file"]) + + + + def test_get_trial_index_cv(self): + try: + os.remove("test_get_trial_index.pkl") + except OSError: + pass + + try: + os.remove("test_get_trial_index.pkl.lock") + except OSError: + pass + + experiment = Experiment.Experiment(".", "test_get_trial_index", folds=5) + params0 = {"x": "1"} + params1 = {"x": "2"} + params2 = {"x": "3"} + params3 = {"x": "4"} + params4 = {"x": "5"} + + trial_index0 = runsolver_wrapper.get_trial_index(experiment, 0, params0) + self.assertEqual(trial_index0, 0) + experiment.set_one_fold_running(trial_index0, 0) + experiment.set_one_fold_complete(trial_index0, 0, 1, 1) + self.assertEqual(trial_index0, + runsolver_wrapper.get_trial_index(experiment, 1, params0)) + experiment.set_one_fold_running(trial_index0, 1) + experiment.set_one_fold_complete(trial_index0, 1, 1, 1) + self.assertEqual(trial_index0, + runsolver_wrapper.get_trial_index(experiment, 2, params0)) + experiment.set_one_fold_running(trial_index0, 2) + experiment.set_one_fold_complete(trial_index0, 2, 1, 1) + self.assertEqual(trial_index0, + runsolver_wrapper.get_trial_index(experiment, 3, params0)) + experiment.set_one_fold_running(trial_index0, 3) + experiment.set_one_fold_complete(trial_index0, 3, 1, 1) + self.assertEqual(trial_index0, + runsolver_wrapper.get_trial_index(experiment, 4, params0)) + experiment.set_one_fold_running(trial_index0, 4) + experiment.set_one_fold_complete(trial_index0, 4, 1, 1) + + trial_index1 = runsolver_wrapper.get_trial_index(experiment, 0, params1) + self.assertEqual(trial_index1, 1) + experiment.set_one_fold_running(trial_index1, 0) + experiment.set_one_fold_complete(trial_index1, 0, 1, 1) + self.assertEqual(trial_index1, + runsolver_wrapper.get_trial_index(experiment, 1, params1)) + experiment.set_one_fold_running(trial_index1, 1) + experiment.set_one_fold_complete(trial_index1, 1, 1, 1) + self.assertEqual(trial_index1, + runsolver_wrapper.get_trial_index(experiment, 2, params1)) + experiment.set_one_fold_running(trial_index1, 2) + experiment.set_one_fold_complete(trial_index1, 2, 1, 1) + self.assertEqual(trial_index1, + runsolver_wrapper.get_trial_index(experiment, 3, params1)) + experiment.set_one_fold_running(trial_index1, 3) + experiment.set_one_fold_complete(trial_index1, 3, 1, 1) + self.assertEqual(trial_index1, + runsolver_wrapper.get_trial_index(experiment, 4, params1)) + experiment.set_one_fold_running(trial_index1, 4) + experiment.set_one_fold_complete(trial_index1, 4, 1, 1) + + trial_index2 = runsolver_wrapper.get_trial_index(experiment, 0, params2) + self.assertEqual(trial_index2, 2) + experiment.set_one_fold_running(trial_index2, 0) + experiment.set_one_fold_complete(trial_index2, 0, 1, 1) + + trial_index3 = runsolver_wrapper.get_trial_index(experiment, 0, params3) + self.assertEqual(trial_index3, 3) + experiment.set_one_fold_running(trial_index3, 0) + experiment.set_one_fold_complete(trial_index3, 0, 1, 1) + + trial_index4 = runsolver_wrapper.get_trial_index(experiment, 0, params4) + self.assertEqual(trial_index4, 4) + experiment.set_one_fold_running(trial_index4, 0) + experiment.set_one_fold_complete(trial_index4, 0, 1, 1) + + self.assertEqual(trial_index2, + runsolver_wrapper.get_trial_index(experiment, 3, params2)) + self.assertEqual(trial_index4, + runsolver_wrapper.get_trial_index(experiment, 4, params4)) + + # Since params1 were already evaluated, this should be a new trial_index + trial_index_test1 = runsolver_wrapper.get_trial_index(experiment, 0, params1) + self.assertEqual(trial_index_test1, 5) + + def test_get_trial_index_nocv(self): + try: + os.remove("test_get_trial_index.pkl") + except OSError: + pass + + try: + os.remove("test_get_trial_index.pkl.lock") + except OSError: + pass + + experiment = Experiment.Experiment(".", "test_get_trial_index", folds=1) + params0 = {"x": "1"} + params1 = {"x": "2"} + params2 = {"x": "3"} + params3 = {"x": "4"} + params4 = {"x": "5"} + + trial_index0 = runsolver_wrapper.get_trial_index(experiment, 0, params0) + self.assertEqual(trial_index0, 0) + experiment.set_one_fold_running(trial_index0, 0) + experiment.set_one_fold_complete(trial_index0, 0, 1, 1) + + trial_index1 = runsolver_wrapper.get_trial_index(experiment, 0, params1) + self.assertEqual(trial_index1, 1) + experiment.set_one_fold_running(trial_index1, 0) + experiment.set_one_fold_complete(trial_index1, 0, 1, 1) + + trial_index2 = runsolver_wrapper.get_trial_index(experiment, 0, params2) + self.assertEqual(trial_index2, 2) + experiment.set_one_fold_running(trial_index2, 0) + experiment.set_one_fold_complete(trial_index2, 0, 1, 1) + + trial_index3 = runsolver_wrapper.get_trial_index(experiment, 0, params3) + self.assertEqual(trial_index3, 3) + experiment.set_one_fold_running(trial_index3, 0) + experiment.set_one_fold_complete(trial_index3, 0, 1, 1) + + trial_index4 = runsolver_wrapper.get_trial_index(experiment, 0, params4) + self.assertEqual(trial_index4, 4) + experiment.set_one_fold_running(trial_index4, 0) + experiment.set_one_fold_complete(trial_index4, 0, 1, 1) + + self.assertEqual(5, + runsolver_wrapper.get_trial_index(experiment, 0, params2)) diff --git a/tests/test_wrapping.py b/tests/test_wrapping.py new file mode 100644 index 00000000..93528a43 --- /dev/null +++ b/tests/test_wrapping.py @@ -0,0 +1,201 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import shutil +import sys +import unittest +import tempfile +import StringIO + +import HPOlib.wrapping as wrapping +import HPOlib.config_parser.parse as parse + +class WrappingTest(unittest.TestCase): + def setUp(self): + # Change into the test directory + os.chdir(os.path.dirname(os.path.realpath(__file__))) + + # Make sure there is no config file + try: + os.remove("./config.cfg") + except: + pass + + @unittest.skip("Not implemented yet") + def test_calculate_wrapping_overhead(self): + self.fail() + + @unittest.skip("Not implemented yet") + def test_calculate_optimizer_time(self): + self.fail() + + # Test spearmint + @unittest.skip("Not implemented yet") + def test_build_spearmint_call(self): + self.fail() + + @unittest.skip("Not implemented yet") + def test_main_spearmint(self): + self.fail() + + # Test tpe + @unittest.skip("Not implemented yet") + def test_build_tpe_call(self): + self.fail() + + @unittest.skip("Not implemented yet") + def test_main_tpe(self): + self.fail() + + # Test random + @unittest.skip("Not implemented yet") + def test_build_random_call(self): + self.fail() + + @unittest.skip("Not implemented yet") + def test_main_random(self): + self.fail() + + # Test smac + @unittest.skip("Not implemented yet") + def test_build_smac_call(self): + self.fail() + + def test_use_option_parser_no_optimizer(self): + # Test not specifying an optimizer but random other options + sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet'] + self.assertRaises(SystemExit, wrapping.use_option_parser) + + def test_use_option_parser_the_right_way(self): + sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet', '-o', 'SMAC'] + args, unknown = wrapping.use_option_parser() + self.assertEqual(args.optimizer, 'SMAC') + self.assertEqual(args.seed, 1) + self.assertEqual(args.title, 'DBNet') + self.assertEqual(len(unknown), 0) + + def test_use_option_parser_with_config(self): + sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet', '-o', 'SMAC', + '--DEFAULT:total_time_limit', '3600'] + args, unknown = wrapping.use_option_parser() + self.assertEqual(len(unknown), 2) + config = parse.parse_config("dummy_config.cfg", + allow_no_value=True, + optimizer_module="smac") + config_args = wrapping.parse_config_values_from_unknown_arguments( + unknown, config) + self.assertEqual(vars(config_args)['DEFAULT:total_time_limit'], + '3600') + + def test_override_config_with_cli_arguments(self): + unknown = ['--DEFAULT:total_time_limit', '50', + '--DEFAULT:numberofjobs', '2'] + config = parse.parse_config("dummy_config.cfg", + allow_no_value=True, + optimizer_module="smac") + config_args = wrapping.parse_config_values_from_unknown_arguments( + unknown, config) + new_config = wrapping.override_config_with_cli_arguments(config, + config_args) + self.assertEqual(new_config.get('DEFAULT', 'numberofjobs'), '2') + + + def test_save_config_to_file(self): + unknown = ['--DEFAULT:total_time_limit', '50', + '--DEFAULT:numberofjobs', '2'] + config = parse.parse_config("dummy_config.cfg", + allow_no_value=True, + optimizer_module="smac") + string_stream = StringIO.StringIO() + wrapping.save_config_to_file(string_stream, config) + file_content = string_stream.getvalue() + asserted_file_content = "[DEFAULT]\n" \ + "numberofjobs = 1\n" \ + "result_on_terminate = 1\n"\ + "function = 1\n"\ + "algorithm = cv.py\n"\ + "run_instance = runsolver_wrapper.py\n"\ + "numbercv = 1\n"\ + "numberofconcurrentjobs = 1\n"\ + "runsolver_time_limit = 3600\n"\ + "total_time_limit = 3600\n"\ + "memory_limit = 2000\n"\ + "cpu_limit = 14400\n"\ + "training_data_format = numpy\n"\ + "use_percentage = 100\n"\ + "max_crash_per_cv = 3\n"\ + "optimizer_version = smac_2_06_01-dev\n"\ + "[SMAC]\n"\ + "numrun = 0\n"\ + "instancefile = train.txt\n"\ + "intrainstanceobj = MEAN\n"\ + "runobj = QUALITY\n"\ + "testinstancefile = test.txt\n"\ + "p = smac/params.pcs\n"\ + "rf_full_tree_bootstrap = False\n"\ + "rf_split_min = 10\n"\ + "adaptivecapping = false\n"\ + "maxincumbentruns = 2000\n"\ + "numiterations = 2147483647\n"\ + "runtimelimit = 2147483647\n"\ + "deterministic = True\n"\ + "retrytargetalgorithmruncount = 0\n"\ + "intensification_percentage = 0\n"\ + "cutofftime = 3700\n"\ + "algoexec = runsolver_wrapper.py\n"\ + "totalnumrunslimit = 1\n"\ + "numconcurrentalgoexecs = 1\n"\ + "numberofjobs = 1\n"\ + "result_on_terminate = 1\n"\ + "function = 1\n"\ + "algorithm = cv.py\n"\ + "run_instance = runsolver_wrapper.py\n"\ + "numbercv = 1\n"\ + "numberofconcurrentjobs = 1\n"\ + "runsolver_time_limit = 3600\n"\ + "total_time_limit = 3600\n"\ + "memory_limit = 2000\n"\ + "cpu_limit = 14400\n"\ + "training_data_format = numpy\n"\ + "use_percentage = 100\n"\ + "max_crash_per_cv = 3\n"\ + "optimizer_version = smac_2_06_01-dev\n" + + self.assertEqual(asserted_file_content, file_content) + string_stream.close() + + @unittest.skip("Not implemented yet") + def test_main_smac(self): + os.chdir("./branin_test/") + sys.argv = ['wrapping.py', 'smac'] + ret = wrapping.main() + self.assertEqual(ret, 0, "Return code of SMAC is not 0, please debug") + + # TODO: Test what happens when the runsolver limit is exceeded + # TODO: Clean up the test directory + + # General main test + @unittest.skip("Not implemented yet") + def test_main(self): + self.fail() + + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/testsuite.py b/tests/testsuite.py new file mode 100644 index 00000000..6c9e2d60 --- /dev/null +++ b/tests/testsuite.py @@ -0,0 +1,28 @@ +import unittest + +import test_benchmark_util +import test_cv +import test_experiment +import test_gridsearch +import test_data_utils +import test_runsolver_wrapper +import test_wrapping + + +def suite(): + _suite = unittest.TestSuite() + _suite.addTest(unittest.makeSuite(test_benchmark_util.BenchmarkUtilTest)) + _suite.addTest(unittest.makeSuite(test_cv.CVTest)) + _suite.addTest(unittest.makeSuite(test_experiment.ExperimentTest)) + _suite.addTest(unittest.makeSuite(test_gridsearch.GridSearchTest)) + _suite.addTest(unittest.makeSuite(test_data_utils.DataUtilTest)) + _suite.addTest(unittest.makeSuite(test_runsolver_wrapper.RunsolverWrapperTest)) + _suite.addTest(unittest.makeSuite(test_wrapping.WrappingTest)) + return _suite + + +if __name__ == "__main__": + runner = unittest.TextTestRunner(verbosity=2) + test_suite = suite() + runner.run(suite()) + From 5b897d7533c2a7871b5b7e99d8aa2f5f4fd7a8c8 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Tue, 18 Feb 2014 15:41:31 +0100 Subject: [PATCH 029/139] REFACTOR: #4, tests added to the setup script --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index dc53738e..729a0084 100644 --- a/setup.py +++ b/setup.py @@ -163,7 +163,7 @@ def run(self): license='LGPLv3', platforms=['Linux'], author=HPOlib.__authors__, - tests_require=['unittest'], + test_suite="tests.testsuite.suite", install_requires=['argparse==1.2.1', 'matplotlib==1.3.1', 'networkx==1.8.1', @@ -183,7 +183,6 @@ def run(self): scripts=scripts, cmdclass={ 'install': InstallRunsolver, - 'test': PyTest }, classifiers=[ 'Programming Language :: Python :: 2.7', From ac2f529a92e3ba41fb626d93d8f70c4bd85461bc Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 18 Feb 2014 18:26:21 +0100 Subject: [PATCH 030/139] FIX: Checking whether benchmark is callable HPOlib does now only accept absolut paths to python modules or paths relative to config.cfg --- HPOlib/check_before_start.py | 34 ++++++++++++---------------------- HPOlib/wrapping.py | 2 +- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index 105722bb..a7513da3 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -83,7 +83,7 @@ def _check_config(experiment_dir): raise Exception("There is no config.cfg in %s" % experiment_dir) -def _check_function(experiment_dir, optimizer_dir): +def _check_function(experiment_dir): # Check whether function exists config_file = os.path.join(experiment_dir, "config.cfg") config = parse_config(config_file, allow_no_value=True) @@ -96,30 +96,20 @@ def _check_function(experiment_dir, optimizer_dir): fn = imp.load_source(fn_name, fn_path) except (ImportError, IOError): print "Could not find algorithm in %s" % fn_path - import traceback - print traceback.format_exc() + import HPOlib.wrapping_util + print HPOlib.wrapping_util.format_traceback(sys.exc_info()) else: - fn = config.get("DEFAULT", "function").replace("../..", "..", 1) - fn_path = os.path.join(optimizer_dir, fn) - fn_path_parent = os.path.join(optimizer_dir, "..", fn) + fn = config.get("DEFAULT", "function") + fn_path = os.path.join(experiment_dir, fn) fn_name, ext = os.path.splitext(os.path.basename(fn_path)) try: + print fn_name, fn_path fn = imp.load_source(fn_name, fn_path) except (ImportError, IOError) as e: - print e - try: - fn = imp.load_source(fn_name, fn_path_parent) - except IOError as e: - print(("Could not find\n%s\n\tin\n%s\n\tor its parent directory " + - "\n%s") - % (fn_name, fn_path, fn_path_parent)) - import traceback - print traceback.format_exc() - sys.exit(1) - except ImportError as e: - import traceback - print traceback.format_exc() - sys.exit(1) + print "Could not find %s in %s" % (fn_name, fn_path) + import HPOlib.wrapping_util + print HPOlib.wrapping_util.format_traceback(sys.exc_info()) + sys.exit(1) del fn @@ -135,10 +125,10 @@ def check_first(experiment_dir): main() -def check_second(experiment_dir, optimizer_dir): +def check_second(experiment_dir): """ Do remaining tests """ print "\talgorithm..", - _check_function(experiment_dir, optimizer_dir) + _check_function(experiment_dir) print "..passed" diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index d3444530..fb71778a 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -240,7 +240,7 @@ def main(): save_config_to_file(f, config) # _check_function - check_before_start.check_second(experiment_dir, optimizer_dir) + check_before_start.check_second(experiment_dir) # initialize/reload pickle file if args.restore: From 0107a0e7e7fbf8713f7dc3a8de49cf4836b25759 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 18 Feb 2014 19:15:12 +0100 Subject: [PATCH 031/139] FIX Check if path to function exists Do not import it as a python module anymore --- HPOlib/benchmarks/branin/config.cfg | 2 +- HPOlib/check_before_start.py | 33 ++++++++--------------------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/HPOlib/benchmarks/branin/config.cfg b/HPOlib/benchmarks/branin/config.cfg index 3b99804a..58b3a695 100644 --- a/HPOlib/benchmarks/branin/config.cfg +++ b/HPOlib/benchmarks/branin/config.cfg @@ -3,5 +3,5 @@ p = smac/params.txt [DEFAULT] function = branin.py -numberOfJobs = 200 +numberOfJobs = 2 result_on_terminate = 1000 diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index a7513da3..d7e73550 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -84,33 +84,18 @@ def _check_config(experiment_dir): def _check_function(experiment_dir): - # Check whether function exists + # Check whether path function exists config_file = os.path.join(experiment_dir, "config.cfg") config = parse_config(config_file, allow_no_value=True) - fn = None - if os.path.isabs(config.get("DEFAULT", "function")): - fn_path = config.get("DEFAULT", "function") - fn_name, ext = os.path.splitext(os.path.basename(fn_path)) - try: - fn = imp.load_source(fn_name, fn_path) - except (ImportError, IOError): - print "Could not find algorithm in %s" % fn_path - import HPOlib.wrapping_util - print HPOlib.wrapping_util.format_traceback(sys.exc_info()) - else: - fn = config.get("DEFAULT", "function") - fn_path = os.path.join(experiment_dir, fn) - fn_name, ext = os.path.splitext(os.path.basename(fn_path)) - try: - print fn_name, fn_path - fn = imp.load_source(fn_name, fn_path) - except (ImportError, IOError) as e: - print "Could not find %s in %s" % (fn_name, fn_path) - import HPOlib.wrapping_util - print HPOlib.wrapping_util.format_traceback(sys.exc_info()) - sys.exit(1) - del fn + path = config.get("DEFAULT", "function") + + if not os.path.isabs(path): + path = os.path.join(experiment_dir, path) + + if not os.path.exists(path): + raise ValueError("Function: %s does not exist" % path) + return def check_zeroth(experiment_dir): From d0c792fabd17707b06a3d968deeb70b0450fe8f3 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Tue, 18 Feb 2014 19:21:31 +0100 Subject: [PATCH 032/139] REFACTOR: #6: Added logging where useful Not Useful when returning the blackbox output string to SMAC --- HPOlib/Experiment.py | 37 +++++++++++------- HPOlib/Locker.py | 11 ++++-- HPOlib/__init__.py | 2 +- HPOlib/benchmark_util.py | 9 ++++- HPOlib/check_before_start.py | 54 ++++++++++++------------- HPOlib/config_parser/parse.py | 21 ++++++---- HPOlib/cv.py | 34 ++++++++-------- HPOlib/data_util.py | 33 +++++++++------- HPOlib/hyperopt_august2013_mod.py | 19 +++++---- HPOlib/randomtpe.py | 6 ++- HPOlib/runsolver_wrapper.py | 24 ++++++------ HPOlib/smac_2_06_01-dev.py | 38 +++++++++--------- HPOlib/spearmint_april2013_mod.py | 22 +++++++---- HPOlib/tpecall.py | 11 ++++-- HPOlib/wrapping.py | 65 ++++++++++++++++++------------- HPOlib/wrapping_util.py | 13 +++++-- 16 files changed, 231 insertions(+), 168 deletions(-) diff --git a/HPOlib/Experiment.py b/HPOlib/Experiment.py index b4b098a1..725ee8e7 100644 --- a/HPOlib/Experiment.py +++ b/HPOlib/Experiment.py @@ -17,6 +17,7 @@ # along with this program. If not, see . import cPickle +import logging import os import scipy import sys @@ -31,6 +32,10 @@ __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" + +logger = logging.getLogger("HPOlib.experiment") + + CANDIDATE_STATE = 0 INCOMPLETE_STATE = 1 RUNNING_STATE = 2 @@ -50,10 +55,10 @@ def __init__(self, expt_dir, expt_name, max_wallclock_time= self.locker = Locker.Locker() # Only one process at a time is allowed to have access to this. - sys.stderr.write("Waiting to lock experiments file " + + logger.info("Waiting to lock experiments file " + self.jobs_pkl + "...") self.locker.lock_wait(self.jobs_pkl) - sys.stderr.write("...acquired\n") + logger.info("...acquired\n") # Does this exist already? if not os.path.exists(self.jobs_pkl): @@ -219,8 +224,8 @@ def set_one_fold_crashed(self, _id, fold, result, duration): self.trials[_id]['instance_status'][fold] = BROKEN_STATE self.trials[_id]['instance_durations'][fold] = duration self.trials[_id]['instance_results'][fold] = result - if (self.get_trial_from_id(_id)['instance_status'] != RUNNING_STATE).all(): - self.get_trial_from_id(_id)['status'] = INCOMPLETE_STATE + if (self.trials[_id]['instance_status'] != RUNNING_STATE).all(): + self.trials[_id]['status'] = INCOMPLETE_STATE self.check_cv_finished(_id) self.total_wallclock_time += duration self._sanity_check() @@ -234,8 +239,8 @@ def set_one_fold_complete(self, _id, fold, result, duration): self.get_trial_from_id(_id)['instance_status'][fold] = COMPLETE_STATE self.get_trial_from_id(_id)['instance_durations'][fold] = duration # Set to incomplete if no job is running - if (self.get_trial_from_id(_id)['instance_status'] != RUNNING_STATE).all(): - self.get_trial_from_id(_id)['status'] = INCOMPLETE_STATE + if (self.trials[_id]['instance_status'] != RUNNING_STATE).all(): + self.trials[_id]['status'] = INCOMPLETE_STATE # Check if all runs are finished self.check_cv_finished(_id) self.total_wallclock_time += duration @@ -259,9 +264,13 @@ def check_cv_finished(self, _id): self.get_trial_from_id(_id)['status'] = BROKEN_STATE else: self.get_trial_from_id(_id)['status'] = COMPLETE_STATE - self.get_trial_from_id(_id)['result'] = np.sum(self.get_trial_from_id(_id)['instance_results']) / self.folds - self.get_trial_from_id(_id)['std'] = np.std(self.get_trial_from_id(_id)['instance_results']) - self.get_trial_from_id(_id)['duration'] = np.sum(self.get_trial_from_id(_id)['instance_durations']) + self.get_trial_from_id(_id)['result'] = \ + np.sum(self.get_trial_from_id(_id)['instance_results'])\ + / self.folds + self.get_trial_from_id(_id)['std'] =\ + np.std(self.get_trial_from_id(_id)['instance_results']) + self.get_trial_from_id(_id)['duration'] =\ + np.sum(self.get_trial_from_id(_id)['instance_durations']) return True else: return False @@ -270,13 +279,13 @@ def check_cv_finished(self, _id): # parameters. Useful to delete all unnecessary entries after a crash in order # to restart def remove_all_but_first_runs(self, restored_runs): - # print "#########Restored runs", restored_runs - # print self.instance_order, len(self.instance_order) + logger.info("Restored runs %d" % restored_runs) + logger.info("%s %s" % (self.instance_order, len(self.instance_order))) if len(self.instance_order) == restored_runs: pass else: for _id, instance in self.instance_order[-1:restored_runs - 1:-1]: - # print "Deleting", _id, instance + logger.info("Deleting %d %d" % (_id, instance)) if np.isfinite(self.trials[_id]['instance_durations'][instance]): self.total_wallclock_time -= \ self.trials[_id]['instance_durations'][instance] @@ -348,7 +357,7 @@ def _sanity_check(self): # Automatically loads this object from a pickle file def _load_jobs(self): - fh = open(self.jobs_pkl, 'r') + fh = open(self.jobs_pkl, 'r') jobs = cPickle.load(fh) fh.close() @@ -385,4 +394,4 @@ def _save_jobs(self): 'trials' : self.trials}, fh) fh.close() cmd = 'mv "%s" "%s"' % (fh.name, self.jobs_pkl) - os.system(cmd) # TODO: Replace with subprocess modules \ No newline at end of file + os.system(cmd) # TODO: Replace with subprocess modules diff --git a/HPOlib/Locker.py b/HPOlib/Locker.py index f78e4d98..72315da1 100755 --- a/HPOlib/Locker.py +++ b/HPOlib/Locker.py @@ -19,13 +19,17 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import logging import os import sys import time +logger = logging.getLogger("HPOlib.locker") + + def safe_delete(filename): - cmd = 'mv "%s" "%s.delete" && rm "%s.delete"' % (filename, filename, + cmd = 'mv "%s" "%s.delete" && rm "%s.delete"' % (filename, filename, filename) fail = os.system(cmd) return not fail @@ -53,13 +57,12 @@ def lock(self, filename): def unlock(self, filename): if not self.locks.has_key(filename): - sys.stderr.write("Trying to unlock not-locked file %s.\n" % - (filename)) + logger.info("Trying to unlock not-locked file %s.\n" % filename) return True if self.locks[filename] == 1: success = safe_delete('%s.lock' % (filename)) if not success: - sys.stderr.write("Could not unlock file: %s.\n" % (filename)) + logger.log("Could not unlock file: %s.\n" % filename) del self.locks[filename] return success else: diff --git a/HPOlib/__init__.py b/HPOlib/__init__.py index 8a99b1bb..e14fe892 100644 --- a/HPOlib/__init__.py +++ b/HPOlib/__init__.py @@ -1,4 +1,4 @@ __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" -__version__="0.0.1" \ No newline at end of file +__version__ = "0.0.1" diff --git a/HPOlib/benchmark_util.py b/HPOlib/benchmark_util.py index 2842ec9f..946acaea 100644 --- a/HPOlib/benchmark_util.py +++ b/HPOlib/benchmark_util.py @@ -16,9 +16,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import logging import sys +logger = logging.getLogger("HPOlib.benchmark_util") + + def parse_cli(): """ Provide a generic command line interface for benchmarks. It will just parse @@ -73,7 +77,8 @@ def parse_cli(): elif arg[0] == "-" and arg[0:2] != "--" and not found_params: raise ValueError("You either try to use arguments with only one lea" "ding minus or try to specify a hyperparameter bef" - "ore the --params argument. %s" % " ".join(cli_args)) + "ore the --params argument. %s" % + " ".join(cli_args)) elif not found_params: raise ValueError("Illegal command line string, expected an argument" @@ -83,4 +88,4 @@ def parse_cli(): raise ValueError("Illegal command line string, expected a hyperpara" "meter starting with - but found %s" % (arg,)) - return args, parameters \ No newline at end of file + return args, parameters diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index 105722bb..a7fec16c 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import logging import os import subprocess import sys @@ -23,6 +24,10 @@ from config_parser.parse import parse_config + +logger = logging.getLogger("HPOlib.check_before_start") + + """This script checks whether all dependencies are installed""" @@ -43,16 +48,16 @@ def _check_modules(): try: import numpy if numpy.__version__ < "1.6.0": - print "WARNING: You are using a numpy %s < 1.6.0. This might not work"\ - % numpy.__version__ + logger.warning("WARNING: You are using a numpy %s < 1.6.0. This " + "might not work" % numpy.__version__) except: raise ImportError("Numpy cannot be imported. Are you sure that it's installed?") try: import scipy if scipy.__version__ < "0.12.0": - print "WARNING: You are using a scipy %s < 0.12.0. This might not work"\ - % scipy.__version__ + logger.warning("WARNING: You are using a scipy %s < 0.12.0. " \ + "This might not work" % scipy.__version__) except: raise ImportError("Scipy cannot be imported. Are you sure that it's installed?") @@ -62,7 +67,8 @@ def _check_modules(): try: import theano except ImportError: - print "Theano not found. You might need this to run some more complex benchmarks!" + logger.warning("Theano not found. You might need this to run some " + "more complex benchmarks!") try: import pymongo @@ -71,9 +77,7 @@ def _check_modules(): raise ImportError("Pymongo cannot be imported. Are you sure it's installed?") if 'cuda' not in os.environ['PATH']: - print "\tCUDA not in $PATH" - # if 'cuda' not in os.environ['LD_LIBRARY_PATH']: - # print "CUDA not in $LD_LIBRARY_PATH" + logger.warning("CUDA not in $PATH") def _check_config(experiment_dir): @@ -95,9 +99,9 @@ def _check_function(experiment_dir, optimizer_dir): try: fn = imp.load_source(fn_name, fn_path) except (ImportError, IOError): - print "Could not find algorithm in %s" % fn_path + logger.error("Could not find algorithm in %s" % fn_path) import traceback - print traceback.format_exc() + logger.error(traceback.format_exc()) else: fn = config.get("DEFAULT", "function").replace("../..", "..", 1) fn_path = os.path.join(optimizer_dir, fn) @@ -106,28 +110,26 @@ def _check_function(experiment_dir, optimizer_dir): try: fn = imp.load_source(fn_name, fn_path) except (ImportError, IOError) as e: - print e + logger.error(e) try: fn = imp.load_source(fn_name, fn_path_parent) except IOError as e: - print(("Could not find\n%s\n\tin\n%s\n\tor its parent directory " + - "\n%s") - % (fn_name, fn_path, fn_path_parent)) + logger.error(("Could not find\n%s\n\tin\n%s\n\tor its parent " + "directory \n%s") % (fn_name, fn_path, fn_path_parent)) import traceback - print traceback.format_exc() + logger.error(traceback.format_exc()) sys.exit(1) except ImportError as e: import traceback - print traceback.format_exc() + logger.error(traceback.format_exc()) sys.exit(1) del fn def check_zeroth(experiment_dir): - print "\tconfig.cfg..", - _check_modules() + logger.info("Check config.cfg..",) _check_config(experiment_dir) - print "..passed" + logger.info("..passed") def check_first(experiment_dir): @@ -137,19 +139,19 @@ def check_first(experiment_dir): def check_second(experiment_dir, optimizer_dir): """ Do remaining tests """ - print "\talgorithm..", + logger.info("Check algorithm..",) _check_function(experiment_dir, optimizer_dir) - print "..passed" + logger.info("..passed") def main(): - print "Checking dependencies:" - print "\tRunsolver..", + logger.info("Check dependencies:") + logger.info("Runsolver..") _check_runsolver() - print "..passed" - print "\tpython_modules..", + logger.info("..passed") + logger.info("Check python_modules..") _check_modules() - print "..passed" + logger.info("..passed") if __name__ == "__main__": diff --git a/HPOlib/config_parser/parse.py b/HPOlib/config_parser/parse.py index 7bc693a7..d1f157a2 100644 --- a/HPOlib/config_parser/parse.py +++ b/HPOlib/config_parser/parse.py @@ -17,11 +17,15 @@ # along with this program. If not, see . import os +import logging import imp import sys import ConfigParser +logger = logging.getLogger("HPOlib.config_parser.parse") + + def parse_config(config_fn, allow_no_value=True, optimizer_module=""): # Reads config_fn # Overwrites with default values from generalDefault.cfg @@ -74,10 +78,10 @@ def parse_config(config_fn, allow_no_value=True, optimizer_module=""): try: optimizer_module_loaded = imp.load_source(optimizer_module_name, optimizer_module_path) except Exception, e: - print('Could not find\n%s\n\tin\n%s\n\t relative to\n%s' + logger.critical('Could not find\n%s\n\tin\n%s\n\t relative to\n%s' % (optimizer_module_name, optimizer_module_path, os.getcwd())) import traceback - print(traceback.format_exc()) + logger.critical(traceback.format_exc()) sys.exit(1) # Add optimizer specific defaults config = optimizer_module_loaded.add_default(config) @@ -87,14 +91,17 @@ def parse_config(config_fn, allow_no_value=True, optimizer_module=""): def main(): config_fn = sys.argv[1] - print 'Read config from %s..' % config_fn - print '\twith spearmint..', + logger.info('Read config from %s..' % config_fn) + logger.info('\twith spearmint..',) parse_config(config_fn, optimizer_module="spearmint") - print '..finished\n\twith smac..', + logger.info('\t..finished') + logger.info('\twith smac..') parse_config(config_fn, optimizer_module="smac") - print '..finished\n\twith tpe..', + logger.info('\t..finished') + logger.info('\twith tpe..') parse_config(config_fn, optimizer_module="tpe") - print '..finished\n..finished' + logger.info('\t..finished') + logger.info('..finished') if __name__ == "__main__": main() diff --git a/HPOlib/cv.py b/HPOlib/cv.py index e936b403..202e1855 100644 --- a/HPOlib/cv.py +++ b/HPOlib/cv.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import logging import os import subprocess import sys @@ -24,19 +25,19 @@ import numpy as np from HPOlib.Experiment import Experiment - from HPOlib.wrapping_util import format_traceback, load_experiment_config_file - __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __license__ = "3-clause BSD License" __contact__ = "automl.org" +logger = logging.getLogger("HPOlib.cv") + + # TODO: This should be in a util function sometime in the future # Is duplicated in runsolver_wrapper.py - def get_optimizer(): return "_".join(os.getcwd().split("/")[-1].split("_")[0:-2]) @@ -48,7 +49,7 @@ def loadExperimentFile(): def doCV(params, folds=10): - print "Starting Cross validation" + logger.info("Starting Cross validation") sys.stdout.flush() optimizer = get_optimizer() cfg = load_experiment_config_file() @@ -60,7 +61,7 @@ def doCV(params, folds=10): results = [] try: - print "###\n", params, "\n###\n" + logger.info("%s" % params) param_array = ["-" + str(param_name) + " " + str(params[param_name]) \ for param_name in params] param_string = " ".join(param_array) @@ -77,17 +78,16 @@ def doCV(params, folds=10): cmd = "%s %d %s %d %d %d %s" % \ (runsolver_wrapper_script, fold, optimizer, 0, 0, 0, \ param_string) - print "Calling command:\n", cmd + logger.info("Calling command:\n%s" % cmd) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, executable="/bin/bash") - print - print "----------------RUNNING RUNSOLVER_WRAPPER-------------------" + logger.info("--------------RUNNING RUNSOLVER_WRAPPER--------------") stdoutdata, stderrdata = process.communicate() if stdoutdata: - print stdoutdata + logger.info(stdoutdata) if stderrdata: - print stderrdata + logger.error(stderrdata) # Read the runsolver_wrapper output lines = stdoutdata.split("\n") @@ -108,8 +108,8 @@ def doCV(params, folds=10): worst_possible = cfg.getfloat("DEFAULT", "result_on_terminate") crashed_runs = np.nansum([0 if res != worst_possible else 1 for res in results]) if crashed_runs >= cfg.getint("DEFAULT", "max_crash_per_cv"): - print "Aborting CV because the number of crashes excceds the " \ - "configured max_crash_per_cv value" + logger.warning("Aborting CV because the number of crashes " \ + "excceds the configured max_crash_per_cv value") return worst_possible # TODO: Error Handling @@ -118,8 +118,8 @@ def doCV(params, folds=10): mean = np.mean(results) except Exception as e: - print format_traceback(sys.exc_info()) - print "CV failed", sys.exc_info()[0], e + logger.error(format_traceback(sys.exc_info())) + logger.error("CV failed %s %s" % (sys.exc_info()[0], e)) # status = "CRASHED" status = "SAT" mean = np.NaN @@ -128,7 +128,7 @@ def doCV(params, folds=10): if not np.isfinite(mean): mean = float(cfg.get("DEFAULT", "result_on_terminate")) - print "Finished CV" + logger.info("Finished CV") return mean @@ -207,7 +207,7 @@ def main(job_id, params): params = flatten_parameter_dict(params) res = doCV(params, folds=folds) - print "Result: ", res + logger.info("Result: %f" % res) # Load the experiment to do time-keeping experiment = loadExperimentFile() @@ -245,4 +245,4 @@ def read_params_from_command_line(): if __name__ == "__main__": params = read_params_from_command_line() - print main(42, params) + main(42, params) diff --git a/HPOlib/data_util.py b/HPOlib/data_util.py index 3f494c5e..1dd555a6 100644 --- a/HPOlib/data_util.py +++ b/HPOlib/data_util.py @@ -18,20 +18,24 @@ import cPickle from gzip import GzipFile as gfile -import numpy as np +import logging import os -import sys + +import numpy as np __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" +logger = logging.getLogger("HPOlib.data_util") + + def load_file(filename, file_format, use_percentage): if not os.path.exists(filename): raise IOError("File %s not found", filename) if file_format == "gfile": - # print "Loading file:", filename + logger.info("Loading file: %s" % filename) fh = gfile(filename, "rb") data = cPickle.load(fh) if use_percentage >= 100.: @@ -40,10 +44,10 @@ def load_file(filename, file_format, use_percentage): max_data = int(len(data) / 100. * use_percentage) data = data[:max_data] fh.close() - # print "Done loading file:", filename, "has", len(data), "datapoints" + logger.info("Done loading file: %s has %d datapoints" % (filename, len(data))) elif file_format == "pickle": - # print "Loading file:", filename + logger.info("Loading file: %s" % filename) fh = open(filename, "r") data = cPickle.load(fh) if use_percentage >= 100.: @@ -51,10 +55,10 @@ def load_file(filename, file_format, use_percentage): else: data = data[:len(data) / 100. * use_percentage] fh.close() - # print "Done loading file:", filename, "has", len(data), "datapoints" + logger.info("Done loading file: %s has %d datapoints" % (filename, len(data))) elif file_format == "numpy": - # print "Loading file:", filename + logger.info("Loading file: %s" % filename) fh = open(filename, "r") data = np.load(fh) if use_percentage >= 100.: @@ -62,7 +66,7 @@ def load_file(filename, file_format, use_percentage): else: data = data[:len(data) / 100. * use_percentage] fh.close() - # print "Done loading file:", filename, "has", len(data), "datapoints" + logger.info("Done loading file: %s has %d datapoints" % (filename, len(data))) else: raise ValueError("%s is an unknown training_data_format", file_format) @@ -84,9 +88,6 @@ def custom_split(data, n_train, n_valid): "data (%d)") % (n_train, n_valid, len(data)) train = data[0:n_train] valid = data[n_train:] - # print type(train) - # print type(valid) - return train, valid @@ -106,7 +107,8 @@ def prepare_cv_for_fold(data, fold, folds): In case no data is handed over to the function, None is returned. """ - # print "Fold", fold, "of ", folds, "folds" + logger.info("Splitting data:") + logger.info("Fold %s of %s folds" % (fold, folds)) # Create an array with the split points if data is not None: data_len = len(data) @@ -124,7 +126,7 @@ def prepare_cv_for_fold(data, fold, folds): data = np.array(data) if isinstance(data, np.ndarray): - cv_split_mask = np.empty((data_len), dtype = np.bool) + cv_split_mask = np.empty((data_len), dtype=np.bool) for i in range(folds): if i != fold: cv_split_mask[splits[i]:splits[i+1]] = 1 @@ -132,7 +134,8 @@ def prepare_cv_for_fold(data, fold, folds): cv_split_mask[splits[i]:splits[i+1]] = 0 train = data[cv_split_mask] valid = data[~cv_split_mask] - # print data.shape, train.shape, valid.shape, train.itemsize + logger.info("%d, %d, %d, %d" % (data.shape, train.shape, valid.shape, + train.itemsize)) else: train = [] valid = [] @@ -141,4 +144,4 @@ def prepare_cv_for_fold(data, fold, folds): valid.append(datum) else: train.append(datum) - return train, valid \ No newline at end of file + return train, valid diff --git a/HPOlib/hyperopt_august2013_mod.py b/HPOlib/hyperopt_august2013_mod.py index dcd37c7b..1df593f2 100644 --- a/HPOlib/hyperopt_august2013_mod.py +++ b/HPOlib/hyperopt_august2013_mod.py @@ -17,12 +17,15 @@ # along with this program. If not, see . import cPickle +import logging import os import sys import HPOlib.wrapping_util as wrapping_util +logger = logging.getLogger("HPOlib.hyperopt_august2013_mod") + version_info = ("# %76s #\n" % "https://github.com/hyperopt/hyperopt/tree/486aebec8a4170e4781d99bbd6cca09123b12717") __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] @@ -53,7 +56,7 @@ def restore(config, optimizer_dir, **kwargs): """ restore_file = os.path.join(optimizer_dir, 'state.pkl') if not os.path.exists(restore_file): - print "Oups, this should have been checked before" + logger.error("Oups, this should have been checked before") raise Exception("%s does not exist" % (restore_file,)) # Special settings for restoring @@ -99,6 +102,8 @@ def main(config, options, experiment_dir, **kwargs): # Find experiment directory if options.restore: if not os.path.exists(options.restore): + logger.error("The restore directory %s does not exist." % options + .restore) raise Exception("The restore directory does not exist") optimizer_dir = options.restore else: @@ -117,13 +122,13 @@ def main(config, options, experiment_dir, **kwargs): if not os.path.exists(os.path.join(optimizer_dir, space)): os.symlink(os.path.join(experiment_dir, optimizer_str, space), os.path.join(optimizer_dir, space)) - sys.stdout.write("### INFORMATION ################################################################\n") - sys.stdout.write("# You're running %40s #\n" % path_to_optimizer) + logger.info("### INFORMATION ""################################################################") + logger.info("# You're running %40s #" % path_to_optimizer) if version == SYSTEM_WIDE: pass else: - sys.stdout.write("# To reproduce our results you need version 0.0.3.dev, which can be found here:#\n") - sys.stdout.write("%s" % version_info) - sys.stdout.write("# A newer version might be available, but not yet built in. #\n") - sys.stdout.write("################################################################################\n") + logger.info("# To reproduce our results you need version 0.0.3.dev, which can be found here:#") + logger.info("%s" % version_info) + logger.info("# A newer version might be available, but not yet built in. #") + logger.info("################################################################################") return cmd, optimizer_dir diff --git a/HPOlib/randomtpe.py b/HPOlib/randomtpe.py index 6080bf83..da212fa1 100644 --- a/HPOlib/randomtpe.py +++ b/HPOlib/randomtpe.py @@ -19,6 +19,7 @@ #!/usr/bin/env python import cPickle +import logging import os import HPOlib.wrapping_util as wrapping_util @@ -27,6 +28,9 @@ __contact__ = "automl.org" +logger = logging.getLogger("HPOlib.randomtpe") + + def buildRandomCall(config, options, optimizer_dir): #For TPE (and Random Search) we have to cd to the exp_dir call = 'cd ' + optimizer_dir + '\n' + \ @@ -44,7 +48,7 @@ def buildRandomCall(config, options, optimizer_dir): def restore(config, optimizer_dir, **kwargs): restore_file = os.path.join(optimizer_dir, 'state.pkl') if not os.path.exists(restore_file): - print "Oups, this should have been checked before" + logger.error("Oups, this should have been checked before") raise Exception("%s does not exist" % (restore_file,)) return -1 diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 9ebb51aa..96b014a6 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -18,6 +18,7 @@ from collections import OrderedDict import imp +import logging import numpy as np import os import re @@ -32,6 +33,9 @@ __contact__ = "automl.org" +logger = logging.getLogger("HPOlib.runsolver_wrapper") + + # TODO: This should be in a util function sometime in the future # Is duplicated in cv.py def get_optimizer(): @@ -192,18 +196,18 @@ def get_function_filename(cfg): try: fn = imp.load_source(fn_name, fn_path) except (ImportError, IOError) as e: - print "Raised", e, "trying to recover..." + logger.warning("Raised" + e.message + "trying to recover...") try: - print os.path.exists(fn_path_parent) + logger.info(os.path.exists(fn_path_parent)) fn = imp.load_source(fn_name, fn_path_parent) except (ImportError, IOError): - print os.path.join(fn_path_parent) - print(("Could not find\n%s\n\tin\n%s\n\tor its parent directory " + - "relative to\n%s") - % (fn_name, fn_path, os.getcwd())) + logger.critical(os.path.join(fn_path_parent)) + logger.critical(("Could not find\n%s\n\tin\n%s\n\tor its parent " + "directory relative to\n%s") + % (fn_name, fn_path, os.getcwd())) import traceback - print traceback.format_exc() + logger.critical(traceback.format_exc()) sys.exit(1) fn_filename = os.path.realpath(fn.__file__) if fn_filename[-3:] == "pyc": @@ -269,7 +273,6 @@ def parse_output_files(cfg, run_instance_output, runsolver_output_file): os.remove(runsolver_output_file) rval = (cpu_time, wallclock_time, "SAT", float(result_array[6].strip(",")), cfg.get("DEFAULT", "function")) - #print result_string else: rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("DEFAULT", @@ -323,9 +326,7 @@ def main(): process = subprocess.Popen(cmd, stdout=fh, stderr=fh, shell=True, executable="/bin/bash") - print - print cmd - print "-----------------------RUNNING RUNSOLVER----------------------------" + logger.info("-----------------------RUNNING RUNSOLVER----------------------------") process.wait() fh.close() @@ -353,6 +354,7 @@ def main(): experiment.end_cv(time.time()) del experiment + logger.info(return_string) print return_string return return_string diff --git a/HPOlib/smac_2_06_01-dev.py b/HPOlib/smac_2_06_01-dev.py index b9253e0b..24d0cdd6 100644 --- a/HPOlib/smac_2_06_01-dev.py +++ b/HPOlib/smac_2_06_01-dev.py @@ -17,6 +17,7 @@ # along with this program. If not, see . import glob +import logging import os import re import subprocess @@ -27,14 +28,16 @@ import HPOlib.wrapping_util as wrapping_util +logger = logging.getLogger("HPOlib.smac_2_06_01-dev") + + __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" path_to_optimizer = "optimizers/smac_2_06_01-dev" -version_info = ("# %76s #\n# %76s #\n# %76s #\n" % - ("Automatic Configurator Library ==> v2.06.01-development-643 (a1f71813a262)", +version_info = ["Automatic Configurator Library ==> v2.06.01-development-643 (a1f71813a262)", "Random Forest Library ==> v1.05.01-development-95 (4a8077e95b21)", - "SMAC ==> v2.06.01-development-620 (9380d2c6bab9)")) + "SMAC ==> v2.06.01-development-620 (9380d2c6bab9)"] optimizer_str = "smac_2_06_01-dev" @@ -44,13 +47,13 @@ def _get_state_run(optimizer_dir): if len(rungroups) == 1: rungroup = rungroups[0] else: - print "Found multiple rungroups, take the newest one." + logger.warning("Found multiple rungroups, take the newest one.") creation_times = [] for i, filename in enumerate(rungroups): creation_times.append(float(os.path.getctime(filename))) newest = np.argmax(creation_times) rungroup = rungroups[newest] - print creation_times, newest, rungroup + logger.info(creation_times, newest, rungroup) state_runs = glob.glob(rungroup + "/state-run*") if len(state_runs) != 1: @@ -128,13 +131,13 @@ def restore(config, optimizer_dir, **kwargs): kwargs['cmd']) smac_cmd = re.sub('--outputDirectory ' + optimizer_dir, '--outputDirectory ' + optimizer_dir + "restart_rungroups", smac_cmd) - print smac_cmd + logger.info(smac_cmd) process = subprocess.Popen(smac_cmd, stdout=fh, stderr=fh, shell=True, executable="/bin/bash") - print "-----------------------RUNNING----------------------------------" + logger.info("----------------------RUNNING--------------------------------") ret = process.wait() fh.close() - print "Finished with return code: " + str(ret) + logger.info("Finished with return code: " + str(ret)) # os.remove("smac_restart.out") # read smac.out and look how many states are restored @@ -144,7 +147,6 @@ def restore(config, optimizer_dir, **kwargs): for line in fh.readlines(): match = prog.search(line) if match: - # print "###########Match", match.group(0), match.group(2) restored_runs = int(match.group(2)) # Find out all rungroups and state-runs @@ -154,18 +156,13 @@ def restore(config, optimizer_dir, **kwargs): state_run_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.getcwd(), state_run) state_runs = glob.glob(state_run_path + "/runs_and_results-it*.csv") - # print state_run_path + "/runs_and_results-it*.csv" - # print state_runs state_run_iterations = [] for state_run in state_runs: match = re.search(r"(runs_and_results-it)([0-9]{1,100})(.csv)", state_run) - # print state_run if match: state_run_iterations.append(float(match.group(2))) - # print state_run_iterations run_and_results_fn = state_runs[np.argmax(state_run_iterations)] - # print run_and_results_fn runs_and_results = open(run_and_results_fn) lines = runs_and_results.readlines() @@ -225,10 +222,11 @@ def main(config, options, experiment_dir, **kwargs): if not os.path.exists(os.path.join(optimizer_dir, params)): os.symlink(os.path.join(experiment_dir, optimizer_str, params), os.path.join(optimizer_dir, params)) - sys.stdout.write("### INFORMATION ################################################################\n") - sys.stdout.write("# You're running %40s #\n" % path_to_optimizer) - sys.stdout.write("%s" % version_info) - sys.stdout.write("# A newer version might be available, but not yet built in. #\n") - sys.stdout.write("# Please use this version only to reproduce our results on automl.org #\n") - sys.stdout.write("################################################################################\n") + logger.info("### INFORMATION ################################################################") + logger.info("# You're running %40s #" % path_to_optimizer) + for v in version_info: + logger.info("# %76s #" % v) + logger.info("# A newer version might be available, but not yet built in. #") + logger.info("# Please use this version only to reproduce our results on automl.org #") + logger.info("################################################################################") return cmd, optimizer_dir \ No newline at end of file diff --git a/HPOlib/spearmint_april2013_mod.py b/HPOlib/spearmint_april2013_mod.py index 70e34d6d..8f4a56b8 100644 --- a/HPOlib/spearmint_april2013_mod.py +++ b/HPOlib/spearmint_april2013_mod.py @@ -17,6 +17,7 @@ # along with this program. If not, see . import cPickle +import logging import os import sys @@ -28,6 +29,10 @@ __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" + +logger = logging.getLogger("HPOlib.spearmint_april2013_mod") + + path_to_optimizer = "optimizers/spearmint_april2013_mod/" version_info = ("# %76s #\n" % "https://github.com/JasperSnoek/spearmint/tree/613350f2f617de3af5f101b1dc5eccf60867f67e") @@ -49,7 +54,7 @@ def build_spearmint_call(config, options, optimizer_dir): '--grid-seed', str(options.seed)]) if config.get('SPEARMINT', 'method') != "GPEIChooser" and \ config.get('SPEARMINT', 'method') != "GPEIOptChooser": - sys.stdout.write('WARNING: This chooser might not work yet\n') + logger.warning('WARNING: This chooser might not work yet\n') call = ' '.join([call, config.get("SPEARMINT", 'method_args')]) return call @@ -62,10 +67,11 @@ def restore(config, optimizer_dir, **kwargs): """ restore_file = os.path.join(optimizer_dir, "expt-grid.pkl") if not os.path.exists(restore_file): - print "Oups, this should have been checked before" + logger.error("Oups, this should have been checked before") raise Exception("%s does not exist" % (restore_file,)) sys.path.append(os.path.join( os.path.dirname(os.path.realpath(__file__)), path_to_optimizer)) + # We need the Grid because otherwise we cannot load the pickle file import ExperimentGrid # Assumes that all not valid states are marked as crashed fh = open(restore_file) @@ -109,10 +115,10 @@ def main(config, options, experiment_dir, **kwargs): if not os.path.exists(os.path.join(optimizer_dir, configpb)): os.symlink(os.path.join(experiment_dir, optimizer_str, configpb), os.path.join(optimizer_dir, configpb)) - sys.stdout.write("### INFORMATION ################################################################\n") - sys.stdout.write("# You're running %40s #\n" % path_to_optimizer) - sys.stdout.write("%s" % version_info) - sys.stdout.write("# A newer version might be available, but not yet built in. #\n") - sys.stdout.write("# Please use this version only to reproduce our results on automl.org #\n") - sys.stdout.write("################################################################################\n") + logger.info("### INFORMATION ################################################################") + logger.info("# You're running %40s #" % path_to_optimizer) + logger.info("%s" % version_info) + logger.info("# A newer version might be available, but not yet built in. #") + logger.info("# Please use this version only to reproduce our results on automl.org #") + logger.info("################################################################################") return cmd, optimizer_dir \ No newline at end of file diff --git a/HPOlib/tpecall.py b/HPOlib/tpecall.py index 945ecce7..8358dc3f 100644 --- a/HPOlib/tpecall.py +++ b/HPOlib/tpecall.py @@ -19,9 +19,10 @@ #!/usr/bin/env python import cPickle -from optparse import OptionParser from functools import partial from importlib import import_module +import logging +from optparse import OptionParser import os import hyperopt @@ -29,6 +30,10 @@ __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" + +logger = logging.getLogger("HPOlib.tpecall") + + """ def pyll_replace_list_with_dict(search_space, indent = 0): "" @@ -90,7 +95,7 @@ def main(): import sys sys.path.append("./") sys.path.append("") - print os.getcwd() + logger.info(os.getcwd()) module = import_module(space) search_space = module.space fn = import_module(algo) @@ -130,7 +135,7 @@ def main(): fh.close() best = trials.argmin - print "Best Value found for params:", best + logger.info("Best Value found for params: " + str(best)) if __name__ == "__main__": main() \ No newline at end of file diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 98206ce3..3b4e9977 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -18,6 +18,7 @@ from argparse import ArgumentParser import imp +import logging import os import subprocess import sys @@ -31,6 +32,13 @@ __contact__ = "automl.org" +logging.basicConfig(format='[%(levelname)s] [%(asctime)s:%(name)s] %(' + 'message)s', datefmt='%H:%M:%S') +hpolib_logger = logging.getLogger("HPOlib") +hpolib_logger.setLevel(logging.INFO) +logger = logging.getLogger("HPOlib.wrapping") + + def calculate_wrapping_overhead(trials): wrapping_time = 0 for times in zip(trials.cv_starttime, trials.cv_endtime): @@ -186,8 +194,8 @@ def main(): # Check whether we're calling a optimizer version, that links to another # e.g. calling tpe, but running hyperopt_august2013_mod if optimizer != config.get('DEFAULT', 'optimizer_version'): - print("[INFO] You called -o %s, but this is only a link to version %s" % - (optimizer, config.get('DEFAULT', 'optimizer_version'))) + logger.warning("You called -o %s, but this is only a link to " + "version %s" % (optimizer, config.get('DEFAULT', 'optimizer_version'))) optimizer = config.get('DEFAULT', 'optimizer_version') # Override config values with command line values @@ -212,9 +220,9 @@ def main(): optimizer_module = imp.load_source(optimizer, wrapping_dir + "/" + optimizer + ".py") except (ImportError, IOError): - print "Optimizer module", optimizer, "not found" + logger.critical("Optimizer module %s not found" % optimizer) import traceback - print traceback.format_exc() + logger.critical(traceback.format_exc()) sys.exit(1) optimizer_call, optimizer_dir = optimizer_module.main(config=config, options=args, @@ -271,12 +279,12 @@ def main(): optimizer_dir=optimizer_dir, cmd=cmd) except: - print "Could not restore runs for %s" % args.restore + logger.critical("Could not restore runs for %s" % args.restore) import traceback - print traceback.format_exc() + logger.critical(traceback.format_exc()) sys.exit(1) - print "Restored %d runs" % restored_runs + logger.info("Restored %d runs" % restored_runs) trials.remove_all_but_first_runs(restored_runs) fh = open(os.path.join(optimizer_dir, optimizer + ".out"), "a") fh.write("#" * 80 + "\n" + "Restart! Restored %d runs.\n" % restored_runs) @@ -294,22 +302,22 @@ def main(): # Run call if args.printcmd: - print cmd + logger.info(cmd) return 0 else: - print cmd + logger.info(cmd) output_file = os.path.join(optimizer_dir, optimizer + ".out") fh = open(output_file, "a") # process = subprocess.Popen(cmd, stdout=fh, stderr=fh, shell=True, executable="/bin/bash") # ret = process.wait() - print "-----------------------RUNNING----------------------------------" + logger.info("-----------------------RUNNING----------------------------------") if args.verbose: # Print std and err output for optimizer proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print 'Optimizer runs with PID (+1):', proc.pid + logger.info('Optimizer runs with PID (+1): %d' % proc.pid) while proc.poll() is None: next_line = proc.stdout.readline() @@ -320,12 +328,12 @@ def main(): elif args.silent: # Print nothing proc = subprocess.Popen(cmd, shell=True, stdin=fh, stdout=fh, stderr=fh) - print 'Optimizer runs with PID (+1):', proc.pid + logger.info('Optimizer runs with PID (+1): %d' % proc.pid) else: # Print only stderr proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=fh, stderr=subprocess.PIPE) - print 'Optimizer runs with PID (+1):', proc.pid + logger.info('Optimizer runs with PID (+1): %d' % proc.pid) while proc.poll() is None: next_line = proc.stderr.readline() @@ -335,7 +343,7 @@ def main(): ret = proc.returncode - print "-----------------------END--------------------------------------" + logger.info("-----------------------END--------------------------------------") trials = Experiment.Experiment(optimizer_dir, optimizer) trials.endtime.append(time.time()) @@ -343,26 +351,27 @@ def main(): trials._save_jobs() # trials.finish_experiment() total_time = 0 - print "### Best result" - print trials.get_best() - print "###\n### Durations" + logger.info("Best result") + logger.info(trials.get_best()) + logger.info("Durations") try: for starttime, endtime in zip(trials.starttime, trials.endtime): total_time += endtime - starttime - print "Needed a total of %f seconds" % total_time - print "The optimizer %s took %10.5f seconds" %\ - (optimizer, float(calculate_optimizer_time(trials))) - print "The overhead of HPOlib is %f seconds" % \ - (calculate_wrapping_overhead(trials)) - print "The benchmark itself took %f seconds" % \ - trials.total_wallclock_time + logger.info("Needed a total of %f seconds" % total_time) + logger.info("The optimizer %s took %10.5f seconds" %\ + (optimizer, float(calculate_optimizer_time(trials)))) + logger.info("The overhead of HPOlib is %f seconds" % \ + (calculate_wrapping_overhead(trials))) + logger.info("The benchmark itself took %f seconds" % \ + trials.total_wallclock_time) except Exception as e: import HPOlib.wrapping_util - print HPOlib.wrapping_util.format_traceback(sys.exc_info()) - print "Experiment itself went fine, " \ - "but calculating durations of optimization failed:", sys.exc_info()[0], e + logger.error(HPOlib.wrapping_util.format_traceback(sys.exc_info())) + logger.error("Experiment itself went fine, but calculating " + "durations of optimization failed: %s %s" % + sys.exc_info()[0], e) del trials - print "###\nFinished with return code: " + str(ret) + logger.info("Finished with return code: " + str(ret)) return ret if __name__ == "__main__": diff --git a/HPOlib/wrapping_util.py b/HPOlib/wrapping_util.py index 8541d5e3..83436562 100644 --- a/HPOlib/wrapping_util.py +++ b/HPOlib/wrapping_util.py @@ -17,6 +17,7 @@ # along with this program. If not, see . import datetime +import logging import math import traceback import os @@ -28,6 +29,9 @@ __contact__ = "automl.org" +logger = logging.getLogger("HPOlib.wrapping_util") + + def get_time_string(): local_time = datetime.datetime.today() time_string = "%d-%d-%d--%d-%d-%d-%d" % (local_time.year, local_time.month, @@ -82,12 +86,13 @@ def load_experiment_config_file(): cfg_filename = "config.cfg" cfg = parse_config(cfg_filename, allow_no_value=True) if not cfg.has_option("DEFAULT", "is_not_original_config_file"): - print "Config file in directory %s seems to be an original config "\ - "which was not created by wrapping.py. Please contact the " \ - "HPOlib maintainer to solve this issue." + logger.critical("Config file in directory %s seems to be an" + " original config which was not created by wrapping.py. " + "Please contact the HPOlib maintainer to solve this issue.") sys.exit(1) return cfg except IOError as e: - print "Could not open config file in directory %s" % os.getcwd() + logger.critical("Could not open config file in directory %s" % + os.getcwd()) sys.exit(1) From d96a13c74562050aae16254a278f11491102c5e2 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 19 Feb 2014 13:39:51 +0100 Subject: [PATCH 033/139] REFACTOR: #14: Make optimizers more indepentend (TPE) All tpe related files are now in HPOlib/optimizers/tpe/ There you find the parser, the *Default.cfg. and the .py. The config tells where to find the optimizer source, which can be everywhere on a system Calling HPO: When you have only one optimizer in tpe HPOlib-run -o tpe of if having multiple optimizers HPOlib-run -o tpe/hyperopt_august2013_mod HPOlib-run -o tpe/hyperopt_aug HPOlib-run -o tpe/anything_not_not_ambiguous --- HPOlib/wrapping.py, HPOlib/wrapping_util.py: * Changes to adjust to new structure * FINALLY renamed optimizer_dir to optimizer_dir_in_experiment HPOlib/check_before_start.py: Add a check for the optimizer, which checks for * optimizer_parser * optimizer_Default.cfg * optimizer.py --- HPOlib/check_before_start.py | 35 ++++++++++ .../hyperopt_august2013_modDefault.cfg | 5 -- HPOlib/config_parser/parse.py | 55 ++++++++-------- HPOlib/config_parser/tpeDefault.cfg | 1 - HPOlib/config_parser/tpe_parser.py | 1 - .../tpe}/hyperopt_august2013_mod.py | 64 +++++++++++-------- .../tpe/hyperopt_august2013_modDefault.cfg | 7 ++ .../tpe}/hyperopt_august2013_mod_parser.py | 30 +++++++-- HPOlib/{ => optimizers/tpe}/tpecall.py | 15 ++--- HPOlib/tpe.py | 1 - HPOlib/wrapping.py | 46 +++++++------ HPOlib/wrapping_util.py | 1 - 12 files changed, 157 insertions(+), 104 deletions(-) delete mode 100644 HPOlib/config_parser/hyperopt_august2013_modDefault.cfg delete mode 120000 HPOlib/config_parser/tpeDefault.cfg delete mode 120000 HPOlib/config_parser/tpe_parser.py rename HPOlib/{ => optimizers/tpe}/hyperopt_august2013_mod.py (67%) create mode 100644 HPOlib/optimizers/tpe/hyperopt_august2013_modDefault.cfg rename HPOlib/{config_parser => optimizers/tpe}/hyperopt_august2013_mod_parser.py (60%) rename HPOlib/{ => optimizers/tpe}/tpecall.py (96%) delete mode 120000 HPOlib/tpe.py diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index c1bef5c9..95c64d9c 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -16,9 +16,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import glob import logging import os import subprocess +import sys from config_parser.parse import parse_config @@ -101,6 +103,39 @@ def _check_function(experiment_dir): return +def check_optimizer(optimizer): + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "optimizers", optimizer) + if os.path.isdir(path): + # User told us, e.g. "tpe" + # Optimizer is in our optimizer directory + # Now check how many versions are present + parser = glob.glob(os.path.join(path, '*_parser.py')) + if len(parser) > 1: + logger.critical("Sorry I don't know which optimizer to use: %s" % parser) + sys.exit(1) + version = parser[0][:-10] + elif len(glob.glob(path + '*_parser.py')) == 1: + parser = glob.glob(path + '*_parser.py') + version = parser[0][:-10] + elif len(glob.glob(path + '*_parser.py')) > 1: + # Note this is a different case + # User told us e.g. "tpe/hyperopt_august" but this was not specific enough + logger.critical("Sorry I don't know which optimizer to use: %s" % glob.glob(path + '*_parser.py')) + sys.exit(1) + else: + logger.critical("We cannot find: %s" % path) + sys.exit(1) + + # Now check the other stuff + if not os.path.exists(version + "Default.cfg"): + logger.critical("Sorry I cannot find the default config for your optimizer: %sDefault.cfg" % version) + sys.exit(1) + if not os.path.exists(version + ".py"): + logger.critical("Sorry I cannot find the script to call your optimizer: %s.py" % version) + else: + return version + + def check_zeroth(experiment_dir): logger.info("Check config.cfg..",) _check_config(experiment_dir) diff --git a/HPOlib/config_parser/hyperopt_august2013_modDefault.cfg b/HPOlib/config_parser/hyperopt_august2013_modDefault.cfg deleted file mode 100644 index 2b578b8f..00000000 --- a/HPOlib/config_parser/hyperopt_august2013_modDefault.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[TPE] -space = space.py -version = hyperopt_august2013_mod -#exp_dir = %(exp_directory)s/tpe/ -#numberEvals = %(numberOfJobs)s \ No newline at end of file diff --git a/HPOlib/config_parser/parse.py b/HPOlib/config_parser/parse.py index d1f157a2..df49a09d 100644 --- a/HPOlib/config_parser/parse.py +++ b/HPOlib/config_parser/parse.py @@ -26,10 +26,10 @@ logger = logging.getLogger("HPOlib.config_parser.parse") -def parse_config(config_fn, allow_no_value=True, optimizer_module=""): +def parse_config(config_fn, allow_no_value=True, optimizer_version=""): # Reads config_fn # Overwrites with default values from generalDefault.cfg - # Loads optimizer specific parser, called 'optimizer_module'_parser.py, which can read its own default config + # Loads optimizer specific parser, called 'optimizer_version'_parser.py, which can read its own default config if not os.path.isfile(config_fn): raise Exception('%s is not a valid file\n' % os.path.join( os.getcwd(), config_fn)) @@ -71,39 +71,42 @@ def parse_config(config_fn, allow_no_value=True, optimizer_module=""): raise Exception('No function specified in .cfg') # Load optimizer parsing module - if optimizer_module == "": + if optimizer_version == "": return config - optimizer_module_name = optimizer_module + "_parser" - optimizer_module_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), optimizer_module_name + ".py") + optimizer_version_parser = optimizer_version + "_parser" + optimizer_version_parser_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), + optimizer_version_parser + ".py") + # noinspection PyBroadException,PyUnusedLocal try: - optimizer_module_loaded = imp.load_source(optimizer_module_name, optimizer_module_path) - except Exception, e: - logger.critical('Could not find\n%s\n\tin\n%s\n\t relative to\n%s' - % (optimizer_module_name, optimizer_module_path, os.getcwd())) + optimizer_module_loaded = imp.load_source(optimizer_version_parser, optimizer_version_parser_path) + except Exception as e: + logger.critical('Could not find\n%s\n\tin\n%s\n\t relative to\n%s' % + (optimizer_version_parser, optimizer_version_parser_path, os.getcwd())) import traceback logger.critical(traceback.format_exc()) sys.exit(1) + # Add optimizer specific defaults config = optimizer_module_loaded.add_default(config) return config - -def main(): - config_fn = sys.argv[1] - logger.info('Read config from %s..' % config_fn) - logger.info('\twith spearmint..',) - parse_config(config_fn, optimizer_module="spearmint") - logger.info('\t..finished') - logger.info('\twith smac..') - parse_config(config_fn, optimizer_module="smac") - logger.info('\t..finished') - logger.info('\twith tpe..') - parse_config(config_fn, optimizer_module="tpe") - logger.info('\t..finished') - logger.info('..finished') - -if __name__ == "__main__": - main() +# TODO: remove, does not seem to be useful anymore +# def main(): +# config_fn = sys.argv[1] +# logger.info('Read config from %s..' % config_fn) +# logger.info('\twith spearmint..',) +# parse_config(config_fn, optimizer_module="spearmint") +# logger.info('\t..finished') +# logger.info('\twith smac..') +# parse_config(config_fn, optimizer_module="smac") +# logger.info('\t..finished') +# logger.info('\twith tpe..') +# parse_config(config_fn, optimizer_module="tpe") +# logger.info('\t..finished') +# logger.info('..finished') +# +# if __name__ == "__main__": +# main() diff --git a/HPOlib/config_parser/tpeDefault.cfg b/HPOlib/config_parser/tpeDefault.cfg deleted file mode 120000 index a3a72127..00000000 --- a/HPOlib/config_parser/tpeDefault.cfg +++ /dev/null @@ -1 +0,0 @@ -hyperopt_august2013_modDefault.cfg \ No newline at end of file diff --git a/HPOlib/config_parser/tpe_parser.py b/HPOlib/config_parser/tpe_parser.py deleted file mode 120000 index f32f95f7..00000000 --- a/HPOlib/config_parser/tpe_parser.py +++ /dev/null @@ -1 +0,0 @@ -hyperopt_august2013_mod_parser.py \ No newline at end of file diff --git a/HPOlib/hyperopt_august2013_mod.py b/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py similarity index 67% rename from HPOlib/hyperopt_august2013_mod.py rename to HPOlib/optimizers/tpe/hyperopt_august2013_mod.py index 1df593f2..074b27a0 100644 --- a/HPOlib/hyperopt_august2013_mod.py +++ b/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py @@ -23,11 +23,9 @@ import HPOlib.wrapping_util as wrapping_util +logger = logging.getLogger("HPOlib.optimizers.tpe.hyperopt_august2013_mod") -logger = logging.getLogger("HPOlib.hyperopt_august2013_mod") - - -version_info = ("# %76s #\n" % "https://github.com/hyperopt/hyperopt/tree/486aebec8a4170e4781d99bbd6cca09123b12717") +version_info = ("# %76s #" % "https://github.com/hyperopt/hyperopt/tree/486aebec8a4170e4781d99bbd6cca09123b12717") __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" @@ -35,7 +33,7 @@ def build_tpe_call(config, options, optimizer_dir): - #For TPE we have to cd to the exp_dir + # For TPE we have to cd to the exp_dir call = "cd " + optimizer_dir + "\n" + \ "python " + os.path.dirname(os.path.realpath(__file__)) + \ "/tpecall.py" @@ -56,7 +54,7 @@ def restore(config, optimizer_dir, **kwargs): """ restore_file = os.path.join(optimizer_dir, 'state.pkl') if not os.path.exists(restore_file): - logger.error("Oups, this should have been checked before") + print "Oups, this should have been checked before" raise Exception("%s does not exist" % (restore_file,)) # Special settings for restoring @@ -82,28 +80,30 @@ def main(config, options, experiment_dir, **kwargs): # **kwargs: Nothing so far time_string = wrapping_util.get_time_string() cmd = "" - SYSTEM_WIDE = 0 - AUGUST_2013_MOD = 1 - - try: - import hyperopt - version = SYSTEM_WIDE - except ImportError: - try: - cmd += "export PYTHONPATH=$PYTHONPATH:" + os.path.dirname(os.path.abspath(__file__)) + \ - "/optimizers/hyperopt_august2013_mod\n" - import optimizers.hyperopt_august2013_mod.hyperopt as hyperopt - except ImportError, e: - import HPOlib.optimizers.hyperopt_august2013_mod.hyperopt as hyperopt - version = AUGUST_2013_MOD - - path_to_optimizer = os.path.abspath(os.path.dirname(hyperopt.__file__)) + + # Add path_to_optimizer to PYTHONPATH and to sys.path + os.environ['PYTHONPATH'] = config.get('TPE', 'path_to_optimizer') + os.pathsep + os.environ['PYTHONPATH'] + sys.path.append(config.get('TPE', 'path_to_optimizer')) + +# TODO: Check whether we might need this again +# SYSTEM_WIDE = 0 +# AUGUST_2013_MOD = 1 + +# try: +# import hyperopt +# version = SYSTEM_WIDE +# except ImportError: +# try: +# cmd += "export PYTHONPATH=$PYTHONPATH:" + os.path.dirname(os.path.abspath(__file__)) + \ +# "/optimizers/hyperopt_august2013_mod\n" +# import optimizers.hyperopt_august2013_mod.hyperopt as hyperopt +# except ImportError, e: +# import HPOlib.optimizers.hyperopt_august2013_mod.hyperopt as hyperopt +# version = AUGUST_2013_MOD # Find experiment directory if options.restore: if not os.path.exists(options.restore): - logger.error("The restore directory %s does not exist." % options - .restore) raise Exception("The restore directory does not exist") optimizer_dir = options.restore else: @@ -122,10 +122,18 @@ def main(config, options, experiment_dir, **kwargs): if not os.path.exists(os.path.join(optimizer_dir, space)): os.symlink(os.path.join(experiment_dir, optimizer_str, space), os.path.join(optimizer_dir, space)) - logger.info("### INFORMATION ""################################################################") - logger.info("# You're running %40s #" % path_to_optimizer) - if version == SYSTEM_WIDE: - pass + + import hyperopt + path_to_loaded_optimizer = os.path.abspath(os.path.dirname(os.path.dirname(hyperopt.__file__))) + + logger.info("### INFORMATION ################################################################") + #logger.info("# You're running %40s #\n" % path_to_optimizer) + if not os.path.samefile(path_to_loaded_optimizer, config.get('TPE', 'path_to_optimizer')): + logger.warning("# You are running: #") + logger.warning("# %76s #" % path_to_loaded_optimizer) + logger.warning("# but hyperopt_august2013_modDefault.cfg says:") + logger.warning("# %76s #" % config.get('TPE', 'path_to_optimizer')) + logger.warning("# Found a global hyperopt version. This installation will be used! #") else: logger.info("# To reproduce our results you need version 0.0.3.dev, which can be found here:#") logger.info("%s" % version_info) diff --git a/HPOlib/optimizers/tpe/hyperopt_august2013_modDefault.cfg b/HPOlib/optimizers/tpe/hyperopt_august2013_modDefault.cfg new file mode 100644 index 00000000..9c50d3af --- /dev/null +++ b/HPOlib/optimizers/tpe/hyperopt_august2013_modDefault.cfg @@ -0,0 +1,7 @@ +[TPE] +space = space.py +version = hyperopt_august2013_mod +# either relative to __file__ or absolute +path_to_optimizer = ./hyperopt_august2013_mod_src +#exp_dir = %(exp_directory)s/tpe/ +#numberEvals = %(numberOfJobs)s \ No newline at end of file diff --git a/HPOlib/config_parser/hyperopt_august2013_mod_parser.py b/HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py similarity index 60% rename from HPOlib/config_parser/hyperopt_august2013_mod_parser.py rename to HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py index cbd75a80..89cdea8d 100644 --- a/HPOlib/config_parser/hyperopt_august2013_mod_parser.py +++ b/HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py @@ -16,23 +16,28 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import logging import os +import sys import ConfigParser +logger = logging.getLogger("HPOlib.optimizers.tpe.hyperopt_august2013_mod_parser") + def add_default(config): - # This module reads tpeDefault.cfg and adds this defaults to a given config + # This module reads __file__Default.cfg and adds this defaults to a given config assert isinstance(config, ConfigParser.RawConfigParser), \ "config is not a valid instance" - config_fn = os.path.join(os.path.dirname(os.path.realpath(__file__)), - "tpeDefault.cfg") - if not os.path.isfile(config_fn): - raise Exception('%s is not a valid file\n' % config_fn) + # Find out name of .cfg, we are in anything_parser.py[c] + optimizer_config_fn = os.path.splitext(__file__)[0][:-7] + "Default.cfg" + if not os.path.exists(optimizer_config_fn): + logger.critical("No default config %s found" % optimizer_config_fn) + sys.exit(1) tpe_config = ConfigParser.SafeConfigParser(allow_no_value=True) - tpe_config.read(config_fn) + tpe_config.read(optimizer_config_fn) # -------------------------------------------------------------------------- # TPE # -------------------------------------------------------------------------- @@ -49,8 +54,19 @@ def add_default(config): config.set('TPE', 'numberEvals', config.get('DEFAULT', 'numberOfJobs')) + path_to_optimizer = tpe_config.get('TPE', 'path_to_optimizer') + if not os.path.isabs(path_to_optimizer): + path_to_optimizer = os.path.join(os.path.dirname(os.path.realpath(__file__)), path_to_optimizer) + + path_to_optimizer = os.path.normpath(path_to_optimizer) + if not os.path.exists(path_to_optimizer): + logger.critical("Path to optimizer not found: %s" % path_to_optimizer) + + config.set('TPE', 'path_to_optimizer', path_to_optimizer) + + # TODO: I don't think we need this anymore # Anyway set this # Makes it possible to call, e.g. hyperopt_august2013_mod via simply tpe - config.set('DEFAULT', 'optimizer_version', tpe_config.get('TPE', 'version')) + # config.set('DEFAULT', 'optimizer_version', tpe_config.get('TPE', 'version')) return config \ No newline at end of file diff --git a/HPOlib/tpecall.py b/HPOlib/optimizers/tpe/tpecall.py similarity index 96% rename from HPOlib/tpecall.py rename to HPOlib/optimizers/tpe/tpecall.py index 8358dc3f..aa0d0a38 100644 --- a/HPOlib/tpecall.py +++ b/HPOlib/optimizers/tpe/tpecall.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + ## # wrapping: A program making it easy to use hyperparameter # optimization software. @@ -16,13 +18,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python - import cPickle +from optparse import OptionParser from functools import partial from importlib import import_module -import logging -from optparse import OptionParser import os import hyperopt @@ -30,10 +29,6 @@ __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" - -logger = logging.getLogger("HPOlib.tpecall") - - """ def pyll_replace_list_with_dict(search_space, indent = 0): "" @@ -95,7 +90,7 @@ def main(): import sys sys.path.append("./") sys.path.append("") - logger.info(os.getcwd()) + print os.getcwd() module = import_module(space) search_space = module.space fn = import_module(algo) @@ -135,7 +130,7 @@ def main(): fh.close() best = trials.argmin - logger.info("Best Value found for params: " + str(best)) + print "Best Value found for params:", best if __name__ == "__main__": main() \ No newline at end of file diff --git a/HPOlib/tpe.py b/HPOlib/tpe.py deleted file mode 120000 index 64428bd1..00000000 --- a/HPOlib/tpe.py +++ /dev/null @@ -1 +0,0 @@ -hyperopt_august2013_mod.py \ No newline at end of file diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 9dc153e4..5d2f8ab9 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -198,52 +198,51 @@ def main(): # Now we can safely import non standard things import numpy as np import HPOlib.Experiment as Experiment + import HPOlib.wrapping_util optimizer = args.optimizer + # Check how many optimizer versions are present + optimizer_version = check_before_start.check_optimizer(optimizer) - config_file = os.path.join(experiment_dir, "config.cfg") - config = parse_config(config_file, allow_no_value=True, optimizer_module=optimizer) + logger.warning("You called -o %s, I am using optimizer defined in %sDefault.cfg" % (optimizer, optimizer_version)) + optimizer = os.path.basename(optimizer_version) - # Check whether we're calling a optimizer version, that links to another - # e.g. calling tpe, but running hyperopt_august2013_mod - if optimizer != config.get('DEFAULT', 'optimizer_version'): - logger.warning("You called -o %s, but this is only a link to " - "version %s" % (optimizer, config.get('DEFAULT', 'optimizer_version'))) - optimizer = config.get('DEFAULT', 'optimizer_version') + config_file = os.path.join(experiment_dir, "config.cfg") + config = parse_config(config_file, allow_no_value=True, optimizer_version=optimizer_version) # Override config values with command line values - config_overrides = parse_config_values_from_unknown_arguments(unknown, - config) + config_overrides = parse_config_values_from_unknown_arguments(unknown, config) config = override_config_with_cli_arguments(config, config_overrides) # Saving the config file is down further at the bottom, as soon as we get # hold of the new optimizer directory wrapping_dir = os.path.dirname(os.path.realpath(__file__)) + # TODO: We don't need this anymore, if we install HPOlib # Try adding runsolver to path os.putenv('PATH', os.environ['PATH'] + ":" + wrapping_dir + "/../runsolver/src/") # _check_runsolver, _check_modules() check_before_start.check_first(experiment_dir) + # TODO: We also don't need this # build call cmd = "export PYTHONPATH=$PYTHONPATH:" + wrapping_dir + "\n" # Load optimizer try: - optimizer_module = imp.load_source(optimizer, wrapping_dir + "/" + - optimizer + ".py") + optimizer_dir = os.path.dirname(os.path.realpath(optimizer_version)) + optimizer_module = imp.load_source(optimizer_dir, optimizer_version + ".py") except (ImportError, IOError): logger.critical("Optimizer module %s not found" % optimizer) import traceback logger.critical(traceback.format_exc()) sys.exit(1) - optimizer_call, optimizer_dir = optimizer_module.main(config=config, + optimizer_call, optimizer_dir_in_experiment = optimizer_module.main(config=config, options=args, - experiment_dir= - experiment_dir) + experiment_dir=experiment_dir) cmd += optimizer_call - with open(os.path.join(optimizer_dir, "config.cfg"), "w") as f: + with open(os.path.join(optimizer_dir_in_experiment, "config.cfg"), "w") as f: config.set("DEFAULT", "is_not_original_config_file", "True") save_config_to_file(f, config) @@ -253,15 +252,15 @@ def main(): # initialize/reload pickle file if args.restore: try: - os.remove(os.path.join(optimizer_dir, optimizer + ".pkl.lock")) + os.remove(os.path.join(optimizer_dir_in_experiment, optimizer + ".pkl.lock")) except OSError: pass folds = config.getint('DEFAULT', 'numberCV') - trials = Experiment.Experiment(optimizer_dir, optimizer, folds=folds, + trials = Experiment.Experiment(optimizer_dir_in_experiment, optimizer, folds=folds, max_wallclock_time=config.get('DEFAULT', 'cpu_limit'), title=args.title) - trials.optimizer = optimizer + trials.optimizer = optimizer_version # TODO: We do not have any old runs anymore. DELETE this! if args.restore: @@ -289,7 +288,7 @@ def main(): #noinspection PyBroadException try: restored_runs = optimizer_module.restore(config=config, - optimizer_dir=optimizer_dir, + optimizer_dir=optimizer_dir_in_experiment, cmd=cmd) except: logger.critical("Could not restore runs for %s" % args.restore) @@ -299,7 +298,7 @@ def main(): logger.info("Restored %d runs" % restored_runs) trials.remove_all_but_first_runs(restored_runs) - fh = open(os.path.join(optimizer_dir, optimizer + ".out"), "a") + fh = open(os.path.join(optimizer_dir_in_experiment, optimizer + ".out"), "a") fh.write("#" * 80 + "\n" + "Restart! Restored %d runs.\n" % restored_runs) fh.close() @@ -319,7 +318,7 @@ def main(): return 0 else: logger.info(cmd) - output_file = os.path.join(optimizer_dir, optimizer + ".out") + output_file = os.path.join(optimizer_dir_in_experiment, optimizer + ".out") fh = open(output_file, "a") # process = subprocess.Popen(cmd, stdout=fh, stderr=fh, shell=True, executable="/bin/bash") # ret = process.wait() @@ -358,7 +357,7 @@ def main(): logger.info("-----------------------END--------------------------------------") - trials = Experiment.Experiment(optimizer_dir, optimizer) + trials = Experiment.Experiment(optimizer_dir_in_experiment, optimizer) trials.endtime.append(time.time()) #noinspection PyProtectedMember trials._save_jobs() @@ -378,7 +377,6 @@ def main(): logger.info("The benchmark itself took %f seconds" % \ trials.total_wallclock_time) except Exception as e: - import HPOlib.wrapping_util logger.error(HPOlib.wrapping_util.format_traceback(sys.exc_info())) logger.error("Experiment itself went fine, but calculating " "durations of optimization failed: %s %s" % diff --git a/HPOlib/wrapping_util.py b/HPOlib/wrapping_util.py index 83436562..5da8e6d5 100644 --- a/HPOlib/wrapping_util.py +++ b/HPOlib/wrapping_util.py @@ -95,4 +95,3 @@ def load_experiment_config_file(): logger.critical("Could not open config file in directory %s" % os.getcwd()) sys.exit(1) - From 61a5faa35af7a30c329997785717636404abcb81 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 19 Feb 2014 14:14:44 +0100 Subject: [PATCH 034/139] REFACTOR: #14: Make optimizers more indepentend (Randomtpe) All tpe related files are now in HPOlib/optimizers/tpe/ There you find the parser, the *Default.cfg. and the .py. The config tells where to find the optimizer source, which can be everywhere on a system Calling HPO with randomtpe: Now we have two optimizers in tpe HPOlib-run -o tpe # will fail We can call HPOlib-run -o tpe/h HPOlib-run -o tpe/r --- also added randomtpe to branin --- .../branin/random_hyperopt_august2013_mod | 1 + .../optimizers/tpe/hyperopt_august2013_mod.py | 4 +- .../tpe/random_hyperopt_august2013_mod.py} | 61 ++++++++++++++----- .../random_hyperopt_august2013_modDefault.cfg | 1 + .../random_hyperopt_august2013_mod_parser.py | 1 + HPOlib/optimizers/tpe/tpecall.py | 9 +++ 6 files changed, 58 insertions(+), 19 deletions(-) create mode 120000 HPOlib/benchmarks/branin/random_hyperopt_august2013_mod rename HPOlib/{randomtpe.py => optimizers/tpe/random_hyperopt_august2013_mod.py} (50%) create mode 120000 HPOlib/optimizers/tpe/random_hyperopt_august2013_modDefault.cfg create mode 120000 HPOlib/optimizers/tpe/random_hyperopt_august2013_mod_parser.py diff --git a/HPOlib/benchmarks/branin/random_hyperopt_august2013_mod b/HPOlib/benchmarks/branin/random_hyperopt_august2013_mod new file mode 120000 index 00000000..398d85cc --- /dev/null +++ b/HPOlib/benchmarks/branin/random_hyperopt_august2013_mod @@ -0,0 +1 @@ +hyperopt_august2013_mod/ \ No newline at end of file diff --git a/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py b/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py index 074b27a0..a2dc9cea 100644 --- a/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py +++ b/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py @@ -29,8 +29,6 @@ __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" -optimizer_str = "hyperopt_august2013_mod" - def build_tpe_call(config, options, optimizer_dir): # For TPE we have to cd to the exp_dir @@ -84,11 +82,11 @@ def main(config, options, experiment_dir, **kwargs): # Add path_to_optimizer to PYTHONPATH and to sys.path os.environ['PYTHONPATH'] = config.get('TPE', 'path_to_optimizer') + os.pathsep + os.environ['PYTHONPATH'] sys.path.append(config.get('TPE', 'path_to_optimizer')) + optimizer_str = os.path.splitext(os.path.basename(__file__))[0] # TODO: Check whether we might need this again # SYSTEM_WIDE = 0 # AUGUST_2013_MOD = 1 - # try: # import hyperopt # version = SYSTEM_WIDE diff --git a/HPOlib/randomtpe.py b/HPOlib/optimizers/tpe/random_hyperopt_august2013_mod.py similarity index 50% rename from HPOlib/randomtpe.py rename to HPOlib/optimizers/tpe/random_hyperopt_august2013_mod.py index da212fa1..856c4528 100644 --- a/HPOlib/randomtpe.py +++ b/HPOlib/optimizers/tpe/random_hyperopt_august2013_mod.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + ## # wrapping: A program making it easy to use hyperparameter # optimization software. @@ -16,11 +18,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python - import cPickle import logging import os +import sys import HPOlib.wrapping_util as wrapping_util @@ -28,18 +29,22 @@ __contact__ = "automl.org" -logger = logging.getLogger("HPOlib.randomtpe") +logger = logging.getLogger("HPOlib.optimizers.tpe.randomtpe") + +version_info = ("# %76s #" % "https://github.com/hyperopt/hyperopt/tree/486aebec8a4170e4781d99bbd6cca09123b12717") +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" -def buildRandomCall(config, options, optimizer_dir): +def build_random_call(config, options, optimizer_dir): #For TPE (and Random Search) we have to cd to the exp_dir call = 'cd ' + optimizer_dir + '\n' + \ - 'python ' + os.path.dirname(os.path.realpath(__file__)) + \ - '/tpecall.py' - call = ' '.join([call, '-p', config.get('TPE', 'space'), \ - '-a', config.get('DEFAULT', 'algorithm'), \ - '-m', config.get('DEFAULT', 'numberOfJobs'), \ - '-s', str(options.seed), '--random']) + 'python ' + os.path.dirname(os.path.realpath(__file__)) + \ + '/tpecall.py' + call = ' '.join([call, '-p', config.get('TPE', 'space'), + '-a', config.get('DEFAULT', 'algorithm'), + '-m', config.get('DEFAULT', 'numberOfJobs'), + '-s', str(options.seed), '--random']) if options.restore: call = ' '.join([call, '-r']) return call @@ -71,18 +76,25 @@ def main(config, options, experiment_dir, **kwargs): # experiment_dir: Experiment directory/Benchmarkdirectory # **kwargs: Nothing so far time_string = wrapping_util.get_time_string() + cmd = "" + + # Add path_to_optimizer to PYTHONPATH and to sys.path + os.environ['PYTHONPATH'] = config.get('TPE', 'path_to_optimizer') + os.pathsep + os.environ['PYTHONPATH'] + sys.path.append(config.get('TPE', 'path_to_optimizer')) + optimizer_str = os.path.splitext(os.path.basename(__file__))[0] # Find experiment directory if options.restore: if not os.path.exists(options.restore): - raise Exception("The restore directory %s does not exist" % (options.restore,)) + raise Exception("The restore directory does not exist") optimizer_dir = options.restore else: - optimizer_dir = os.path.join(experiment_dir, "randomtpe_" + \ - str(options.seed) + "_" + time_string) + optimizer_dir = os.path.join(experiment_dir, optimizer_str + "_" + + str(options.seed) + "_" + + time_string) # Build call - cmd = buildRandomCall(config, options, optimizer_dir) + cmd = build_random_call(config, options, optimizer_dir) # Set up experiment directory if not os.path.exists(optimizer_dir): @@ -90,7 +102,24 @@ def main(config, options, experiment_dir, **kwargs): space = config.get('TPE', 'space') # Copy the hyperopt search space if not os.path.exists(os.path.join(optimizer_dir, space)): - os.symlink(os.path.join(experiment_dir, "tpe", space), - os.path.join(optimizer_dir, space)) + os.symlink(os.path.join(experiment_dir, optimizer_str, space), + os.path.join(optimizer_dir, space)) + + import hyperopt + path_to_loaded_optimizer = os.path.abspath(os.path.dirname(os.path.dirname(hyperopt.__file__))) + + logger.info("### INFORMATION ################################################################") + logger.info("# You are running: #") + logger.info("# %76s #" % path_to_loaded_optimizer) + if not os.path.samefile(path_to_loaded_optimizer, config.get('TPE', 'path_to_optimizer')): + logger.warning("# BUT random_hyperopt_august2013_modDefault.cfg says:") + logger.warning("# %76s #" % config.get('TPE', 'path_to_optimizer')) + logger.warning("# Found a global hyperopt version. This installation will be used! #") + else: + logger.info("# To reproduce our results you need version 0.0.3.dev, which can be found here:#") + logger.info("%s" % version_info) + logger.info("# A newer version might be available, but not yet built in. #") + logger.info("################################################################################") + return cmd, optimizer_dir return cmd, optimizer_dir \ No newline at end of file diff --git a/HPOlib/optimizers/tpe/random_hyperopt_august2013_modDefault.cfg b/HPOlib/optimizers/tpe/random_hyperopt_august2013_modDefault.cfg new file mode 120000 index 00000000..a3a72127 --- /dev/null +++ b/HPOlib/optimizers/tpe/random_hyperopt_august2013_modDefault.cfg @@ -0,0 +1 @@ +hyperopt_august2013_modDefault.cfg \ No newline at end of file diff --git a/HPOlib/optimizers/tpe/random_hyperopt_august2013_mod_parser.py b/HPOlib/optimizers/tpe/random_hyperopt_august2013_mod_parser.py new file mode 120000 index 00000000..f32f95f7 --- /dev/null +++ b/HPOlib/optimizers/tpe/random_hyperopt_august2013_mod_parser.py @@ -0,0 +1 @@ +hyperopt_august2013_mod_parser.py \ No newline at end of file diff --git a/HPOlib/optimizers/tpe/tpecall.py b/HPOlib/optimizers/tpe/tpecall.py index aa0d0a38..5adf057f 100644 --- a/HPOlib/optimizers/tpe/tpecall.py +++ b/HPOlib/optimizers/tpe/tpecall.py @@ -22,10 +22,13 @@ from optparse import OptionParser from functools import partial from importlib import import_module +import logging import os import hyperopt +logger = logging.getLogger("HPOlib.optimizers.tpe.tpecall") + __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" @@ -82,6 +85,10 @@ def main(): help="Use a random search") (options, args) = parser.parse_args() + if not os.path.exists(options.spaceFile): + logger.critical("Search space not found: %s" % options.spaceFile) + sys.exit(1) + # First remove ".py" algo, ext = os.path.splitext(os.path.basename(options.algoExec)) space, ext = os.path.splitext(os.path.basename(options.spaceFile)) @@ -91,6 +98,7 @@ def main(): sys.path.append("./") sys.path.append("") print os.getcwd() + module = import_module(space) search_space = module.space fn = import_module(algo) @@ -99,6 +107,7 @@ def main(): if options.random: # We use a random search tpe_with_seed = partial(hyperopt.tpe.rand.suggest, seed=int(options.seed)) + logger.info("Using Random Search") else: tpe_with_seed = partial(hyperopt.tpe.suggest, seed=int(options.seed)) From 2219fabf81a573fdacf3e615916de9a1deff6b60 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 19 Feb 2014 15:15:34 +0100 Subject: [PATCH 035/139] REFACTOR: #14: Make optimizers more indepentend (SMAC) All smac related files are now in HPOlib/optimizers/smac/ There you find the parser, the *Default.cfg. and the .py. The config tells where to find the optimizer source, which can be everywhere on a system Calling HPO: When you have only one optimizer in smac HPOlib-run -o smac of if having multiple optimizers HPOlib-run -o smac/smac-v2_06_02-partial38 or HPOlib-run -o smac/anything_not_not_ambiguous --- HPOlib/benchmarks/branin/config.cfg: * Changed searchspace for smac --- HPOlib/benchmarks/branin/config.cfg | 4 +-- HPOlib/benchmarks/branin/smac | 1 - HPOlib/benchmarks/branin/tpe | 1 - HPOlib/config_parser/smacDefault.cfg | 1 - HPOlib/config_parser/smac_parser.py | 1 - .../{ => optimizers/smac}/smac_2_06_01-dev.py | 15 ++++++---- .../smac}/smac_2_06_01-devDefault.cfg | 5 ++-- .../smac}/smac_2_06_01-dev_parser.py | 28 +++++++++++++------ HPOlib/optimizers/smac_2_06_01-dev/dummy.py | 0 .../optimizers/tpe/hyperopt_august2013_mod.py | 7 ++--- .../tpe/hyperopt_august2013_modDefault.cfg | 8 +++--- HPOlib/smac.py | 1 - HPOlib/wrapping.py | 2 +- 13 files changed, 42 insertions(+), 32 deletions(-) delete mode 120000 HPOlib/benchmarks/branin/smac delete mode 120000 HPOlib/benchmarks/branin/tpe delete mode 120000 HPOlib/config_parser/smacDefault.cfg delete mode 120000 HPOlib/config_parser/smac_parser.py rename HPOlib/{ => optimizers/smac}/smac_2_06_01-dev.py (95%) rename HPOlib/{config_parser => optimizers/smac}/smac_2_06_01-devDefault.cfg (84%) rename HPOlib/{config_parser => optimizers/smac}/smac_2_06_01-dev_parser.py (75%) delete mode 100644 HPOlib/optimizers/smac_2_06_01-dev/dummy.py delete mode 120000 HPOlib/smac.py diff --git a/HPOlib/benchmarks/branin/config.cfg b/HPOlib/benchmarks/branin/config.cfg index 58b3a695..bf069411 100644 --- a/HPOlib/benchmarks/branin/config.cfg +++ b/HPOlib/benchmarks/branin/config.cfg @@ -1,7 +1,7 @@ [SMAC] -p = smac/params.txt +p = smac_2_06_01-dev/params.txt [DEFAULT] function = branin.py -numberOfJobs = 2 +numberOfJobs = 200 result_on_terminate = 1000 diff --git a/HPOlib/benchmarks/branin/smac b/HPOlib/benchmarks/branin/smac deleted file mode 120000 index cd8179ec..00000000 --- a/HPOlib/benchmarks/branin/smac +++ /dev/null @@ -1 +0,0 @@ -smac_2_06_01-dev \ No newline at end of file diff --git a/HPOlib/benchmarks/branin/tpe b/HPOlib/benchmarks/branin/tpe deleted file mode 120000 index 398d85cc..00000000 --- a/HPOlib/benchmarks/branin/tpe +++ /dev/null @@ -1 +0,0 @@ -hyperopt_august2013_mod/ \ No newline at end of file diff --git a/HPOlib/config_parser/smacDefault.cfg b/HPOlib/config_parser/smacDefault.cfg deleted file mode 120000 index 930268c8..00000000 --- a/HPOlib/config_parser/smacDefault.cfg +++ /dev/null @@ -1 +0,0 @@ -smac_2_06_01-devDefault.cfg \ No newline at end of file diff --git a/HPOlib/config_parser/smac_parser.py b/HPOlib/config_parser/smac_parser.py deleted file mode 120000 index 42644afb..00000000 --- a/HPOlib/config_parser/smac_parser.py +++ /dev/null @@ -1 +0,0 @@ -smac_2_06_01-dev_parser.py \ No newline at end of file diff --git a/HPOlib/smac_2_06_01-dev.py b/HPOlib/optimizers/smac/smac_2_06_01-dev.py similarity index 95% rename from HPOlib/smac_2_06_01-dev.py rename to HPOlib/optimizers/smac/smac_2_06_01-dev.py index 24d0cdd6..57adec67 100644 --- a/HPOlib/smac_2_06_01-dev.py +++ b/HPOlib/optimizers/smac/smac_2_06_01-dev.py @@ -34,12 +34,11 @@ __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" -path_to_optimizer = "optimizers/smac_2_06_01-dev" version_info = ["Automatic Configurator Library ==> v2.06.01-development-643 (a1f71813a262)", "Random Forest Library ==> v1.05.01-development-95 (4a8077e95b21)", "SMAC ==> v2.06.01-development-620 (9380d2c6bab9)"] -optimizer_str = "smac_2_06_01-dev" +#optimizer_str = "smac_2_06_01-dev" def _get_state_run(optimizer_dir): @@ -64,7 +63,10 @@ def _get_state_run(optimizer_dir): def build_smac_call(config, options, optimizer_dir): - call = os.path.dirname(os.path.realpath(__file__)) + "/" + path_to_optimizer + "/smac" + import HPOlib + algo_exec_dir = os.path.dirname(HPOlib.__file__) + + call = config.get('SMAC', 'path_to_optimizer') + "/smac" call = " ".join([call, '--numRun', str(options.seed), '--scenario-file', os.path.join(optimizer_dir, 'scenario.txt'), '--cutoffTime', config.get('SMAC', 'cutoffTime'), @@ -74,8 +76,7 @@ def build_smac_call(config, options, optimizer_dir): '--intraInstanceObj', config.get('SMAC', 'intraInstanceObj'), '--runObj', config.get('SMAC', 'runObj'), # '--testInstanceFile', config.get('SMAC', 'testInstanceFile'), - '--algoExec', '"python', os.path.dirname(os.path.realpath(__file__)) + "/" + - config.get('SMAC', 'algoExec') + '"', + '--algoExec', '"python', os.path.join(algo_exec_dir, config.get('SMAC', 'algoExec')) + '"', '--execDir', optimizer_dir, '-p', config.get('SMAC', 'p'), # The experiment dir MUST not be specified when restarting, it is set @@ -187,6 +188,8 @@ def main(config, options, experiment_dir, **kwargs): # **kwargs: Nothing so far time_string = wrapping_util.get_time_string() + optimizer_str = os.path.splitext(os.path.basename(__file__))[0] + # Find experiment directory if options.restore: if not os.path.exists(options.restore): @@ -223,7 +226,7 @@ def main(config, options, experiment_dir, **kwargs): os.symlink(os.path.join(experiment_dir, optimizer_str, params), os.path.join(optimizer_dir, params)) logger.info("### INFORMATION ################################################################") - logger.info("# You're running %40s #" % path_to_optimizer) + logger.info("# You're running %40s #" % config.get('SMAC', 'path_to_optimizer')) for v in version_info: logger.info("# %76s #" % v) logger.info("# A newer version might be available, but not yet built in. #") diff --git a/HPOlib/config_parser/smac_2_06_01-devDefault.cfg b/HPOlib/optimizers/smac/smac_2_06_01-devDefault.cfg similarity index 84% rename from HPOlib/config_parser/smac_2_06_01-devDefault.cfg rename to HPOlib/optimizers/smac/smac_2_06_01-devDefault.cfg index 4ba62d6c..65c31f67 100644 --- a/HPOlib/config_parser/smac_2_06_01-devDefault.cfg +++ b/HPOlib/optimizers/smac/smac_2_06_01-devDefault.cfg @@ -14,7 +14,7 @@ runObj = QUALITY instanceFile = train.txt testInstanceFile = test.txt #execDir = ./ -p = smac/params.pcs +p = smac_2_06_01-dev/params.pcs rf_full_tree_bootstrap = False rf_split_min = 10 @@ -27,4 +27,5 @@ deterministic = True retryTargetAlgorithmRunCount = 0 intensification_percentage = 0 -version = smac_2_06_01-dev \ No newline at end of file +# either relative to __file__ or absolute +path_to_optimizer = ./smac_2_06_01-dev_src diff --git a/HPOlib/config_parser/smac_2_06_01-dev_parser.py b/HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py similarity index 75% rename from HPOlib/config_parser/smac_2_06_01-dev_parser.py rename to HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py index 4b108e64..ac3f4e2b 100644 --- a/HPOlib/config_parser/smac_2_06_01-dev_parser.py +++ b/HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py @@ -16,10 +16,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import logging import os +import sys import ConfigParser +logger = logging.getLogger("HPOlib.optimizers.smac.smac_2_06_01-dev_parser") + def add_default(config): # This module reads smacDefault.cfg and adds this defaults to a given config @@ -27,13 +31,14 @@ def add_default(config): assert isinstance(config, ConfigParser.RawConfigParser), \ 'config is not a valid instance' - config_fn = os.path.join(os.path.dirname(os.path.realpath(__file__)), - "smacDefault.cfg") - if not os.path.isfile(config_fn): - raise Exception('%s is not a valid file\n' % config_fn) + # Find out name of .cfg, we are in anything_parser.py[c] + optimizer_config_fn = os.path.splitext(__file__)[0][:-7] + "Default.cfg" + if not os.path.exists(optimizer_config_fn): + logger.critical("No default config %s found" % optimizer_config_fn) + sys.exit(1) smac_config = ConfigParser.SafeConfigParser(allow_no_value=True) - smac_config.read(config_fn) + smac_config.read(optimizer_config_fn) # -------------------------------------------------------------------------- # SMAC # -------------------------------------------------------------------------- @@ -66,7 +71,14 @@ def add_default(config): config.set('SMAC', 'numConcurrentAlgoExecs', config.get('DEFAULT', 'numberOfConcurrentJobs')) - # Anyway set this - # Makes it possible to call, e.g. smac_2_06_01-dev via smac - config.set('DEFAULT', 'optimizer_version', smac_config.get('SMAC', 'version')) + path_to_optimizer = smac_config.get('SMAC', 'path_to_optimizer') + if not os.path.isabs(path_to_optimizer): + path_to_optimizer = os.path.join(os.path.dirname(os.path.realpath(__file__)), path_to_optimizer) + + path_to_optimizer = os.path.normpath(path_to_optimizer) + if not os.path.exists(path_to_optimizer): + logger.critical("Path to optimizer not found: %s" % path_to_optimizer) + + config.set('SMAC', 'path_to_optimizer', path_to_optimizer) + return config \ No newline at end of file diff --git a/HPOlib/optimizers/smac_2_06_01-dev/dummy.py b/HPOlib/optimizers/smac_2_06_01-dev/dummy.py deleted file mode 100644 index e69de29b..00000000 diff --git a/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py b/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py index a2dc9cea..6e6822f9 100644 --- a/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py +++ b/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py @@ -125,11 +125,10 @@ def main(config, options, experiment_dir, **kwargs): path_to_loaded_optimizer = os.path.abspath(os.path.dirname(os.path.dirname(hyperopt.__file__))) logger.info("### INFORMATION ################################################################") - #logger.info("# You're running %40s #\n" % path_to_optimizer) + logger.info("# You are running: #") + logger.info("# %76s #" % path_to_loaded_optimizer) if not os.path.samefile(path_to_loaded_optimizer, config.get('TPE', 'path_to_optimizer')): - logger.warning("# You are running: #") - logger.warning("# %76s #" % path_to_loaded_optimizer) - logger.warning("# but hyperopt_august2013_modDefault.cfg says:") + logger.warning("# BUT hyperopt_august2013_modDefault.cfg says:") logger.warning("# %76s #" % config.get('TPE', 'path_to_optimizer')) logger.warning("# Found a global hyperopt version. This installation will be used! #") else: diff --git a/HPOlib/optimizers/tpe/hyperopt_august2013_modDefault.cfg b/HPOlib/optimizers/tpe/hyperopt_august2013_modDefault.cfg index 9c50d3af..df580f1b 100644 --- a/HPOlib/optimizers/tpe/hyperopt_august2013_modDefault.cfg +++ b/HPOlib/optimizers/tpe/hyperopt_august2013_modDefault.cfg @@ -1,7 +1,7 @@ [TPE] space = space.py -version = hyperopt_august2013_mod -# either relative to __file__ or absolute -path_to_optimizer = ./hyperopt_august2013_mod_src #exp_dir = %(exp_directory)s/tpe/ -#numberEvals = %(numberOfJobs)s \ No newline at end of file +#numberEvals = %(numberOfJobs)s + +# either relative to __file__ or absolute +path_to_optimizer = ./hyperopt_august2013_mod_src \ No newline at end of file diff --git a/HPOlib/smac.py b/HPOlib/smac.py deleted file mode 120000 index 6b7db85c..00000000 --- a/HPOlib/smac.py +++ /dev/null @@ -1 +0,0 @@ -smac_2_06_01-dev.py \ No newline at end of file diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 5d2f8ab9..0e2cc349 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -355,7 +355,7 @@ def main(): ret = proc.returncode - logger.info("-----------------------END--------------------------------------") + logger.info("\n-----------------------END--------------------------------------") trials = Experiment.Experiment(optimizer_dir_in_experiment, optimizer) trials.endtime.append(time.time()) From 60695e364d7937a33f81938e06e365e2dfb6aa1d Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 19 Feb 2014 15:33:50 +0100 Subject: [PATCH 036/139] REFACTOR: #14: Make optimizers more indepentend (SPEARMINT) All spearmint related files are now in HPOlib/optimizers/spearmint/ There you find the parser, the *Default.cfg. and the .py. The config tells where to find the optimizer source, which can be everywhere on a system Calling HPO with spearmint: When you have only one version inf spearmint/ HPOlib-run -o spearmint or if having multiple optimizers HPOlib-run -o spearmint/spearmint_april2013_mod HPOlib-run -o tpe/anything_not_not_ambiguous --- From now on you can place the optimizer folder containing: * optimizer sources * optimizer_parser.py * optimizerDefault.cfg * optimizer.py anywhere. Call your optimizer from experiment dir: HPOlib-run -o /absolute/path/to/optimizer/folder[/maybe_some_version_info] --- HPOlib/benchmarks/branin/spearmint | 1 - HPOlib/config_parser/spearmintDefault.cfg | 1 - HPOlib/config_parser/spearmint_parser.py | 1 - .../spearmint}/spearmint_april2013_mod.py | 11 +++---- .../spearmint_april2013_modDefault.cfg | 3 +- .../spearmint_april2013_mod_parser.py | 33 ++++++++++++++----- .../spearmint_april2013_mod_src}/__init__.py | 0 .../optimizers/tpe/hyperopt_august2013_mod.py | 2 ++ HPOlib/spearmint.py | 1 - 9 files changed, 33 insertions(+), 20 deletions(-) delete mode 120000 HPOlib/benchmarks/branin/spearmint delete mode 120000 HPOlib/config_parser/spearmintDefault.cfg delete mode 120000 HPOlib/config_parser/spearmint_parser.py rename HPOlib/{ => optimizers/spearmint}/spearmint_april2013_mod.py (94%) rename HPOlib/{config_parser => optimizers/spearmint}/spearmint_april2013_modDefault.cfg (87%) rename HPOlib/{config_parser => optimizers/spearmint}/spearmint_april2013_mod_parser.py (64%) rename HPOlib/optimizers/{spearmint_april2013_mod => spearmint/spearmint_april2013_mod_src}/__init__.py (100%) delete mode 120000 HPOlib/spearmint.py diff --git a/HPOlib/benchmarks/branin/spearmint b/HPOlib/benchmarks/branin/spearmint deleted file mode 120000 index 525a4844..00000000 --- a/HPOlib/benchmarks/branin/spearmint +++ /dev/null @@ -1 +0,0 @@ -spearmint_april2013_mod/ \ No newline at end of file diff --git a/HPOlib/config_parser/spearmintDefault.cfg b/HPOlib/config_parser/spearmintDefault.cfg deleted file mode 120000 index aa1cdf36..00000000 --- a/HPOlib/config_parser/spearmintDefault.cfg +++ /dev/null @@ -1 +0,0 @@ -spearmint_april2013_modDefault.cfg \ No newline at end of file diff --git a/HPOlib/config_parser/spearmint_parser.py b/HPOlib/config_parser/spearmint_parser.py deleted file mode 120000 index a6032ea6..00000000 --- a/HPOlib/config_parser/spearmint_parser.py +++ /dev/null @@ -1 +0,0 @@ -spearmint_april2013_mod_parser.py \ No newline at end of file diff --git a/HPOlib/spearmint_april2013_mod.py b/HPOlib/optimizers/spearmint/spearmint_april2013_mod.py similarity index 94% rename from HPOlib/spearmint_april2013_mod.py rename to HPOlib/optimizers/spearmint/spearmint_april2013_mod.py index 8f4a56b8..59529f0e 100644 --- a/HPOlib/spearmint_april2013_mod.py +++ b/HPOlib/optimizers/spearmint/spearmint_april2013_mod.py @@ -23,7 +23,7 @@ import numpy as np -import HPOlib.wrapping_util as wrapping_util +import HPOlib.wrapping_util __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] @@ -37,12 +37,10 @@ version_info = ("# %76s #\n" % "https://github.com/JasperSnoek/spearmint/tree/613350f2f617de3af5f101b1dc5eccf60867f67e") -optimizer_str = "spearmint_april2013_mod" - def build_spearmint_call(config, options, optimizer_dir): - call = 'python ' + os.path.dirname(os.path.realpath(__file__)) + \ - "/" + path_to_optimizer + '/spearmint_sync.py' + print + call = 'python ' + os.path.join(config.get('SPEARMINT', 'path_to_optimizer'), 'spearmint_sync.py') call = ' '.join([call, optimizer_dir, '--config', config.get('SPEARMINT', 'config'), '--max-concurrent', config.get('DEFAULT', 'numberOfConcurrentJobs'), @@ -93,7 +91,8 @@ def main(config, options, experiment_dir, **kwargs): # experiment_dir: Experiment directory/Benchmark_directory # **kwargs: Nothing so far - time_string = wrapping_util.get_time_string() + time_string = HPOlib.wrapping_util.get_time_string() + optimizer_str = os.path.splitext(os.path.basename(__file__))[0] # Find experiment directory if options.restore: diff --git a/HPOlib/config_parser/spearmint_april2013_modDefault.cfg b/HPOlib/optimizers/spearmint/spearmint_april2013_modDefault.cfg similarity index 87% rename from HPOlib/config_parser/spearmint_april2013_modDefault.cfg rename to HPOlib/optimizers/spearmint/spearmint_april2013_modDefault.cfg index 7c5c636b..15599fe5 100644 --- a/HPOlib/config_parser/spearmint_april2013_modDefault.cfg +++ b/HPOlib/optimizers/spearmint/spearmint_april2013_modDefault.cfg @@ -20,4 +20,5 @@ spearmint_polling_time = 3.0 max_concurrent = 1 #max_finished_jobs = %(numberOfJobs)s -version = spearmint_april2013_mod \ No newline at end of file +# either relative to __file__ or absolute +path_to_optimizer = ./spearmint_april2013_mod_src diff --git a/HPOlib/config_parser/spearmint_april2013_mod_parser.py b/HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py similarity index 64% rename from HPOlib/config_parser/spearmint_april2013_mod_parser.py rename to HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py index d34cd277..e07fe390 100644 --- a/HPOlib/config_parser/spearmint_april2013_mod_parser.py +++ b/HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py @@ -16,21 +16,29 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import logging import os +import sys import ConfigParser +logger = logging.getLogger("HPOlib.optimizers.spearmint.spearmint_april2013_mod_parser") + def add_default(config): - # This module reads spearmintDefault.cfg and adds this defaults to a given config - assert isinstance(config, ConfigParser.RawConfigParser), "config is not a valid instance" + # This module reads __file__Default.cfg and adds this defaults to a given config + assert isinstance(config, ConfigParser.RawConfigParser), \ + "config is not a valid instance" - config_fn = os.path.join(os.path.dirname(os.path.realpath(__file__)), "spearmintDefault.cfg") - if not os.path.isfile(config_fn): - raise Exception('%s is not a valid file\n' % config_fn) + # Find out name of .cfg, we are in anything_parser.py[c] + optimizer_config_fn = os.path.splitext(__file__)[0][:-7] + "Default.cfg" + if not os.path.exists(optimizer_config_fn): + logger.critical("No default config %s found" % optimizer_config_fn) + sys.exit(1) spearmint_config = ConfigParser.SafeConfigParser(allow_no_value=True) - spearmint_config.read(config_fn) + spearmint_config.read(optimizer_config_fn) + # -------------------------------------------------------------------------- # SPEARMINT # -------------------------------------------------------------------------- @@ -56,7 +64,14 @@ def add_default(config): config.set('SPEARMINT', 'max_finished_jobs', config.get('DEFAULT', 'numberOfJobs')) - # Anyway set this - # Makes it possible to call, e.g. smac_2_06_01-dev via smac - config.set('DEFAULT', 'optimizer_version', spearmint_config.get('SPEARMINT', 'version')) + path_to_optimizer = spearmint_config.get('SPEARMINT', 'path_to_optimizer') + if not os.path.isabs(path_to_optimizer): + path_to_optimizer = os.path.join(os.path.dirname(os.path.realpath(__file__)), path_to_optimizer) + + path_to_optimizer = os.path.normpath(path_to_optimizer) + if not os.path.exists(path_to_optimizer): + logger.critical("Path to optimizer not found: %s" % path_to_optimizer) + + config.set('SPEARMINT', 'path_to_optimizer', path_to_optimizer) + return config \ No newline at end of file diff --git a/HPOlib/optimizers/spearmint_april2013_mod/__init__.py b/HPOlib/optimizers/spearmint/spearmint_april2013_mod_src/__init__.py similarity index 100% rename from HPOlib/optimizers/spearmint_april2013_mod/__init__.py rename to HPOlib/optimizers/spearmint/spearmint_april2013_mod_src/__init__.py diff --git a/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py b/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py index 6e6822f9..0cb3e7f8 100644 --- a/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py +++ b/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py @@ -80,8 +80,10 @@ def main(config, options, experiment_dir, **kwargs): cmd = "" # Add path_to_optimizer to PYTHONPATH and to sys.path + # Only for HYPEROPT os.environ['PYTHONPATH'] = config.get('TPE', 'path_to_optimizer') + os.pathsep + os.environ['PYTHONPATH'] sys.path.append(config.get('TPE', 'path_to_optimizer')) + optimizer_str = os.path.splitext(os.path.basename(__file__))[0] # TODO: Check whether we might need this again diff --git a/HPOlib/spearmint.py b/HPOlib/spearmint.py deleted file mode 120000 index b96d4d3f..00000000 --- a/HPOlib/spearmint.py +++ /dev/null @@ -1 +0,0 @@ -spearmint_april2013_mod.py \ No newline at end of file From 33659a4028ff3c3d19a231ccef4a11ab5ff725ec Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 20 Feb 2014 13:41:46 +0100 Subject: [PATCH 037/139] REFACTOR: cleaning up gridsearch references and logging --- HPOlib/Experiment.py | 20 +++--- HPOlib/cv.py | 5 +- .../hyperopt_august2013_mod/__init__.py | 0 .../spearmint_april2013_mod_src/__init__.py | 21 ------ HPOlib/runsolver_wrapper.py | 4 ++ tests/test_gridsearch.py | 67 ------------------- tests/testsuite.py | 2 - 7 files changed, 18 insertions(+), 101 deletions(-) delete mode 100644 HPOlib/optimizers/hyperopt_august2013_mod/__init__.py delete mode 100755 HPOlib/optimizers/spearmint/spearmint_april2013_mod_src/__init__.py delete mode 100644 tests/test_gridsearch.py diff --git a/HPOlib/Experiment.py b/HPOlib/Experiment.py index 725ee8e7..814062c3 100644 --- a/HPOlib/Experiment.py +++ b/HPOlib/Experiment.py @@ -25,7 +25,7 @@ import numpy as np -import HPOlib.Locker as Locker +# import HPOlib.Locker as Locker import HPOlib.wrapping_util as wrapping_util @@ -52,13 +52,13 @@ def __init__(self, expt_dir, expt_name, max_wallclock_time= folds = 1 self.jobs_pkl = os.path.join(expt_dir, expt_name + ".pkl") - self.locker = Locker.Locker() + #self.locker = Locker.Locker() # Only one process at a time is allowed to have access to this. - logger.info("Waiting to lock experiments file " + - self.jobs_pkl + "...") - self.locker.lock_wait(self.jobs_pkl) - logger.info("...acquired\n") + #logger.info("Waiting to lock experiments file " + + # self.jobs_pkl + "...") + #self.locker.lock_wait(self.jobs_pkl) + #logger.info("...acquired\n") # Does this exist already? if not os.path.exists(self.jobs_pkl): @@ -124,10 +124,10 @@ def create_trial(self): def __del__(self): self._save_jobs() - if self.locker.unlock(self.jobs_pkl): - sys.stderr.write("Released lock on job grid.\n") - else: - raise Exception("Could not release lock on job grid.\n") + #if self.locker.unlock(self.jobs_pkl): + # sys.stderr.write("Released lock on job grid.\n") + #else: + # raise Exception("Could not release lock on job grid.\n") def result_array(self): result = np.array([trial['result'] for trial in self.trials]) diff --git a/HPOlib/cv.py b/HPOlib/cv.py index 202e1855..274145c7 100644 --- a/HPOlib/cv.py +++ b/HPOlib/cv.py @@ -32,7 +32,10 @@ __license__ = "3-clause BSD License" __contact__ = "automl.org" - +logging.basicConfig(format='[%(levelname)s] [%(asctime)s:%(name)s] %(' + 'message)s', datefmt='%H:%M:%S') +hpolib_logger = logging.getLogger("HPOlib") +hpolib_logger.setLevel(logging.INFO) logger = logging.getLogger("HPOlib.cv") diff --git a/HPOlib/optimizers/hyperopt_august2013_mod/__init__.py b/HPOlib/optimizers/hyperopt_august2013_mod/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/HPOlib/optimizers/spearmint/spearmint_april2013_mod_src/__init__.py b/HPOlib/optimizers/spearmint/spearmint_april2013_mod_src/__init__.py deleted file mode 100755 index 4b9bc013..00000000 --- a/HPOlib/optimizers/spearmint/spearmint_april2013_mod_src/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -## -# Copyright (C) 2012 Jasper Snoek, Hugo Larochelle and Ryan P. Adams -# -# This code is written for research and educational purposes only to -# supplement the paper entitled -# "Practical Bayesian Optimization of Machine Learning Algorithms" -# by Snoek, Larochelle and Adams -# Advances in Neural Information Processing Systems, 2012 -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 96b014a6..09c0c016 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -33,6 +33,10 @@ __contact__ = "automl.org" +logging.basicConfig(format='[%(levelname)s] [%(asctime)s:%(name)s] %(' + 'message)s', datefmt='%H:%M:%S') +hpolib_logger = logging.getLogger("HPOlib") +hpolib_logger.setLevel(logging.INFO) logger = logging.getLogger("HPOlib.runsolver_wrapper") diff --git a/tests/test_gridsearch.py b/tests/test_gridsearch.py deleted file mode 100644 index e9e3974b..00000000 --- a/tests/test_gridsearch.py +++ /dev/null @@ -1,67 +0,0 @@ -from collections import OrderedDict -import unittest -import os - -import HPOlib.optimizers.gridsearch.gridsearch as gridsearch - - -class GridSearchTest(unittest.TestCase): - def setUp(self): - self.hyperparameters = OrderedDict() - self.hyperparameters["x"] = [-5, 0, 5, 10] - self.hyperparameters["y"] = [0, 5, 10, 15] - - def test_parse_hyperopt_string(self): - hyperparameter_string = "x {-5, 0, 5, 10}\ny {0, 5, 10, 15}" - expected = OrderedDict([["x", ["-5", "0", "5", "10"]], - ["y", ["0", "5", "10", "15"]]]) - ret = gridsearch.parse_hyperparameter_string(hyperparameter_string) - self.assertEqual(ret, expected) - - hyperparameter_string = "x {-5, 0, 5, 10} [5]\ny {0, 5, 10, 15}" - ret = gridsearch.parse_hyperparameter_string(hyperparameter_string) - self.assertEqual(ret, expected) - - hyperparameter_string = "x {-5, 0, 5, 10}\ny {0, 5, 10, 15} [5]" - ret = gridsearch.parse_hyperparameter_string(hyperparameter_string) - self.assertEqual(ret, expected) - - hyperparameter_string = "x {-5, 0, 5, 10}\ny 0, 5, 10, 15} [5]" - self.assertRaises(ValueError, gridsearch.parse_hyperparameter_string, - hyperparameter_string) - - def test_construct_cli_call(self): - cli_call = gridsearch.construct_cli_call("cv.py", {"x": -5, "y": 0}) - self.assertEqual(cli_call, "cv.py -x \"'-5'\" -y \"'0'\"") - - - def test_build_grid(self): - grid = gridsearch.build_grid(self.hyperparameters) - self.assertIsInstance(grid, list) - self.assertEqual(len(grid), 16) - self.assertDictEqual(grid[0], {"x": -5, "y": 0}) - self.assertDictEqual(grid[3], {"x": -5, "y": 15}) - self.assertDictEqual(grid[9], {"x": 5, "y": 5}) - self.assertDictEqual(grid[15], {"x": 10, "y": 15}) - - def test_build_grid_3d(self): - self.hyperparameters["z"] = [5, 10, 15, 20] - grid = gridsearch.build_grid(self.hyperparameters) - self.assertIsInstance(grid, list) - self.assertEqual(len(grid), 64) - self.assertDictEqual(grid[0], {"x": -5, "y": 0, "z": 5}) - self.assertDictEqual(grid[16], {"x": 0, "y": 0, "z": 5}) - - def test_perform_grid_search(self): - from numpy import pi, cos - - def branin(params): - x = params["x"] - y = params["y"] - result = (y-(5.1/(4*pi**2))*x**2+5*x/pi-6)**2 - result += 10*(1-1/(8*pi))*cos(x)+10 - return result - - grid = gridsearch.build_grid(self.hyperparameters) - gridsearch.perform_gridsearch(branin, grid) - diff --git a/tests/testsuite.py b/tests/testsuite.py index 6c9e2d60..a8986923 100644 --- a/tests/testsuite.py +++ b/tests/testsuite.py @@ -3,7 +3,6 @@ import test_benchmark_util import test_cv import test_experiment -import test_gridsearch import test_data_utils import test_runsolver_wrapper import test_wrapping @@ -14,7 +13,6 @@ def suite(): _suite.addTest(unittest.makeSuite(test_benchmark_util.BenchmarkUtilTest)) _suite.addTest(unittest.makeSuite(test_cv.CVTest)) _suite.addTest(unittest.makeSuite(test_experiment.ExperimentTest)) - _suite.addTest(unittest.makeSuite(test_gridsearch.GridSearchTest)) _suite.addTest(unittest.makeSuite(test_data_utils.DataUtilTest)) _suite.addTest(unittest.makeSuite(test_runsolver_wrapper.RunsolverWrapperTest)) _suite.addTest(unittest.makeSuite(test_wrapping.WrappingTest)) From 119a4105797f83e6fc1816b9b287a0d46471bf88 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 24 Feb 2014 13:20:07 +0100 Subject: [PATCH 038/139] REFACTOR: #16, #1: refactor Trace with Std * uses now argparse * easier up interface * plots std also on logscale --- HPOlib/Plotting/plotTraceWithStd.py | 288 +++++++++++++--------------- 1 file changed, 133 insertions(+), 155 deletions(-) diff --git a/HPOlib/Plotting/plotTraceWithStd.py b/HPOlib/Plotting/plotTraceWithStd.py index e7f54711..bfd70023 100644 --- a/HPOlib/Plotting/plotTraceWithStd.py +++ b/HPOlib/Plotting/plotTraceWithStd.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + ## # wrapping: A program making it easy to use hyperparameter # optimization software. @@ -16,181 +18,156 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python - +from argparse import ArgumentParser import cPickle import itertools -import optparse -import os import sys -import matplotlib.pyplot as plt -import matplotlib.gridspec as gridspec +from matplotlib.pyplot import tight_layout, figure, subplots_adjust, subplot, savefig, show +import matplotlib.gridspec import numpy as np __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" -def plot_optimization_trace(trialList, nameList, optimum=0, title="", \ - log=True, save="", Ymax=0, Ymin=0): +def plot_optimization_trace(trial_list, name_list, optimum=0, title="", + log=True, save="", y_max=0, y_min=0, scale_std=1): markers = itertools.cycle(['o', 's', 'x', '^']) colors = itertools.cycle(['b', 'g', 'r', 'k']) linestyles = itertools.cycle(['-']) size = 1 - labels = list() - plotH = list() ratio = 5 - gs = gridspec.GridSpec(ratio,1) - fig = plt.figure(1, dpi=100) + gs = matplotlib.gridspec.GridSpec(ratio, 1) + fig = figure(1, dpi=100) fig.suptitle(title, fontsize=16) - ax1 = plt.subplot(gs[0:ratio, :]) + ax1 = subplot(gs[0:ratio, :]) ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5) - minVal = sys.maxint - maxVal = 0 - maxTrials = 0 + min_val = sys.maxint + max_val = -sys.maxint + max_trials = 0 - trialListMeans = list() - trialListStd = list() - trialListMax = list() - trialListMin = list() + trial_list_means = list() + trial_list_std = list() # One trialList represents all runs from one optimizer - for i in range(len(trialList)): - trialListMeans.append(np.mean(np.array(trialList[i]), axis=0)) - trialListStd.append(np.std(np.array(trialList[i]), axis=0)) - trialListMax.append(np.max(np.array(trialList[i]), axis=0)) - trialListMin.append(np.min(np.array(trialList[i]), axis=0)) + for i in range(len(trial_list)): + if log: + trial_list_means.append(np.log10(np.mean(np.array(trial_list[i]), axis=0))) + else: + trial_list_means.append(np.mean(np.array(trial_list[i]), axis=0)) + trial_list_std.append(np.std(np.array(trial_list[i]), axis=0)*scale_std) fig.suptitle(title, fontsize=16) # Plot the average error and std - for i in range(len(trialListMeans)): - x = range(1,len(trialListMeans[i])+1) - y = trialListMeans[i] - optimum + for i in range(len(trial_list_means)): + x = range(1, len(trial_list_means[i])+1) + y = trial_list_means[i] - optimum m = markers.next() c = colors.next() l = linestyles.next() - if log == False: - stdUp = y+trialListStd[i] - stdDown = y-trialListStd[i] - ax1.fill_between(x, y, stdUp, \ - facecolor=c, alpha=0.3, edgecolor=c) - ax1.fill_between(x, y, stdDown, \ - facecolor=c, alpha=0.3, edgecolor=c) - #ax1.errorbar(x, y, yerr=trialListStd[i], color='k') - else: - sys.stdout.write("INFO: Plotting std on a logscale makes no sense\n") - ax1.plot(x, y, color=c, linewidth=size, label=nameList[i][0], \ - linestyle=l, marker=m) - - if min(y) < minVal: - minVal = min(y) - if max(y) > maxVal: - maxVal = max(y) - if len(trialListMeans[i]) > maxTrials: - maxTrials = len(trialListMeans[i]) + std_up = y + trial_list_std[i] + std_down = y - trial_list_std[i] + ax1.fill_between(x, std_down, std_up, + facecolor=c, alpha=0.3, edgecolor=c) + ax1.plot(x, y, color=c, linewidth=size, + label=name_list[i][0] + "(" + str(len(trial_list[i])) + ")", + linestyle=l, marker=m) + if min(std_down) < min_val: + min_val = min(std_down) + if max(std_up) > max_val: + max_val = max(std_up) + if len(trial_list_means[i]) > max_trials: + max_trials = len(trial_list_means[i]) # Maybe plot on logscale - if float(minVal) > 0.0 and log: - ax1.set_yscale('log') + if scale_std != 1: + ylabel = ", %s * std" % scale_std + else: + ylabel = "" + + if log: + ax1.set_ylabel("log10(Minfunction value)" + ylabel) else: - sys.stdout.write("Plotting on logscale not possible or desired (minVal: %s)\n" % - (minVal,)) - log = False + ax1.set_ylabel("Minfunction value" + ylabel) # Descript and label the stuff leg = ax1.legend(loc='best', fancybox=True) leg.get_frame().set_alpha(0.5) ax1.set_xlabel("#Function evaluations") - ax1.set_ylabel("Minfunction value") - if Ymax == Ymin: - ax1.set_ylim([minVal-0.1, maxVal+1]) + + if y_max == y_min: + # Set axes limits + ax1.set_ylim([min_val-0.1*abs((max_val-min_val)), max_val+0.1*abs((max_val-min_val))]) else: - ax1.set_ylim([Ymin, Ymax]) - ax1.set_xlim([0, 100])# maxTrials]) - - plt.tight_layout() - plt.subplots_adjust(top=0.85) + ax1.set_ylim([y_min, y_max]) + ax1.set_xlim([0, 100]) # maxTrials]) + + tight_layout() + subplots_adjust(top=0.85) if save != "": - plt.savefig(save, dpi=100, facecolor='w', edgecolor='w', - orientation='portrait', papertype=None, format=None, - transparent=False, bbox_inches="tight", pad_inches=0.1) + savefig(save, dpi=100, facecolor='w', edgecolor='w', + orientation='portrait', papertype=None, format=None, + transparent=False, bbox_inches="tight", pad_inches=0.1) else: - plt.show() - + show() + + def main(): - parser = optparse.OptionParser() - # Options for specific functions - parser.add_option("-b", "--branin", dest="branin", action="store_true", - default = False, help="Automatic shift for branin function") - parser.add_option("-c", "--camel", dest="camelback", action="store_true", - default = False, help="Automatic shift for camelback function") - parser.add_option("-i", "--har3", dest="har3", action="store_true", - default = False, help="Automatic shift for hartmann3 function") - parser.add_option("-j", "--har6", dest="har6", action="store_true", - default = False, help="Automatic shift for hartmann6 function") - parser.add_option("-g", "--gold", dest="gold", action="store_true", - default = False, help="Automatic shift for goldstein function") - parser.add_option("-o", "--optimum", type=float, dest="optimum", - default=0, help="Optimum") - + prog = "%proc WhatIsThis [WhatIsThis ]" + description = "Plot a Trace with std for multiple experiments" + + parser = ArgumentParser(description=description, prog=prog) + + # Options for specific benchmarks + parser.add_argument("-o", "--optimum", type=float, dest="optimum", + default=0, help="If not set, the optimum is supposed to be zero") + # Options which are available only for this plot - parser.add_option("-a", "--autofill", dest="autofill", default=False, \ - action="store_true", help="Fill trace automatically") - parser.add_option("--fill", dest="fill", type=int, default=0, - help="In case that a trace is not long enough, fill it with the" \ - + " best result so far until there are entries") - + parser.add_argument("-a", "--autofill", action="store_true", dest="autofill", + default=False, help="Fill trace automatically") + parser.add_argument("-c", "--scale", type=float, dest="scale", + default=1, help="Multiply std to get a nicer plot") # General Options - parser.add_option("-l", "--nolog", dest="log", action="store_true", \ - default=False, help="Do NOT plot on log scale") - parser.add_option("--max", dest="max", action="store", default=0, - type=float, help="Maximum of the plot") - parser.add_option("--min", dest="min", action="store", default=0, - type=float, help="Minimum of the plot") - parser.add_option("-s", "--save", dest = "save", default="", \ - help = "Where to save plot instead of showing it?") - parser.add_option("-t", "--title", dest="title", \ - default="", help="Optional supertitle for plot") - - (opts, args) = parser.parse_args() + parser.add_argument("-l", "--nolog", action="store_true", dest="log", + default=False, help="Do NOT plot on log scale") + parser.add_argument("--max", dest="max", type=float, + default=0, help="Maximum of the plot") + parser.add_argument("--min", dest="min", type=float, + default=0, help="Minimum of the plot") + parser.add_argument("-s", "--save", dest="save", + default="", help="Where to save plot instead of showing it?") + parser.add_argument("-t", "--title", dest="title", + default="", help="Optional supertitle for plot") + + args, unknown = parser.parse_known_args() sys.stdout.write("Generate plot for ") - optimum = opts.optimum - title = opts.title - if opts.branin: - optimum = 0.397887 - sys.stdout.write("branin") - if opts.camelback: - optimum = -1.031628453 - sys.stdout.write("camelback") - if opts.har3: - optimum = -3.86278 - sys.stdout.write("hartmann3") - if opts.har6: - optimum = -3.32237 - sys.stdout.write("hartmann6") - if opts.gold: - optimum = 3 - sys.stdout.write("goldstein") - - sys.stdout.write("\nFound " + str(len(args)) + " arguments...") - nameList = list() - trialList = list() - lenList = list() - for i in range(len(args)): - if not ".pkl" in args[i]: - print "Adding", args[i] - nameList.append([args[i],0]) - trialList.append(list()) - lenList.append(0) + optimum = args.optimum + title = args.title + + sys.stdout.write("\nFound " + str(len(unknown)) + " arguments...") + name_list = list() + trial_list = list() + len_list = list() + now_data = False + for i in range(len(unknown)): + if not ".pkl" in unknown[i] and now_data: + raise ValueError("You need at least on .pkl file per Experiment, %s has none" % name_list[-1]) + elif not ".pkl" in unknown[i] and not now_data: + print "Adding", unknown[i] + name_list.append([unknown[i], 0]) + trial_list.append(list()) + len_list.append(0) + now_data = True continue else: - nameList[-1][1] += 1 + now_data = False + name_list[-1][1] += 1 - result_file = args[i] + result_file = unknown[i] fh = open(result_file, "r") trials = cPickle.load(fh) fh.close() @@ -202,33 +179,34 @@ def main(): if result < currentbest: currentbest = result trace.append(currentbest) - """ - # Make sure that trace is long enough - if opts.fill != 0 and opts.fill > len(trace): - print "Pickle file", args[i], "does only contain", len(trace), "entries" - trace.extend([currentbest for i in range(opts.fill - len(trace))]) - print len(trace) - """ - if len(trace) > lenList[-1]: - lenList[-1] = len(trace) - trialList[-1].append(np.array(trace)) - - if opts.autofill: - assert(len(trialList) == len(lenList)) - print lenList - for i in range(len(trialList)): - for t in range(len(trialList[i])): - if len(trialList[i][t]) < lenList[i]: - diff = lenList[i] - len(trialList[i][t]) - trialList[i][t] = np.append(trialList[i][t], [trialList[i][t][-1] for x in range(diff)]) - assert len(trialList[i][t]) == lenList[i], \ - "%s, %s" % (len(trialList[i][t]), lenList[i]) - - plot_optimization_trace(trialList, nameList, optimum, title=title, \ - log=not opts.log, save=opts.save, Ymin=opts.min, Ymax=opts.max) - - if opts.save != "": - sys.stdout.write("Saved plot to " + opts.save + "\n") + if len(trace) > len_list[-1]: + len_list[-1] = len(trace) + trial_list[-1].append(np.array(trace)) + + if now_data: + raise ValueError("You need at least on .pkl file per Experiment, %s has none" % name_list[-1]) + + if args.autofill: + assert(len(trial_list) == len(len_list)) + for i in range(len(trial_list)): + for t in range(len(trial_list[i])): + if len(trial_list[i][t]) < len_list[i]: + diff = len_list[i] - len(trial_list[i][t]) + # noinspection PyUnusedLocal + trial_list[i][t] = np.append(trial_list[i][t], [trial_list[i][t][-1] for x in range(diff)]) + assert len(trial_list[i][t]) == len_list[i], \ + "%s, %s" % (len(trial_list[i][t]), len_list[i]) + else: + for i in range(len(trial_list)): + len_trials = len(trial_list[i][0]) + for t in trial_list[i]: + if len(t) != len_trials: + raise ValueError("(%s != %s), Traces do not have the same length, please use -a" % (str(len(t)), str(len_trials))) + plot_optimization_trace(trial_list, name_list, optimum, title=title, log=not args.log, + save=args.save, y_min=args.min, y_max=args.max, scale_std=args.scale) + + if args.save != "": + sys.stdout.write("Saved plot to " + args.save + "\n") else: sys.stdout.write("..Done\n") From ce97aef4a6d0c55bc41cb3857607451544e38561 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 24 Feb 2014 16:32:06 +0100 Subject: [PATCH 039/139] REFACTOR: #1 use one method to load data for plotting --- HPOlib/Plotting/__init__.py | 1 + HPOlib/Plotting/plotTraceWithStd.py | 81 ++++++++++------------------- HPOlib/Plotting/plot_util.py | 59 +++++++++++++++++++++ 3 files changed, 88 insertions(+), 53 deletions(-) create mode 100644 HPOlib/Plotting/__init__.py create mode 100644 HPOlib/Plotting/plot_util.py diff --git a/HPOlib/Plotting/__init__.py b/HPOlib/Plotting/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/HPOlib/Plotting/__init__.py @@ -0,0 +1 @@ + diff --git a/HPOlib/Plotting/plotTraceWithStd.py b/HPOlib/Plotting/plotTraceWithStd.py index bfd70023..a91f1947 100644 --- a/HPOlib/Plotting/plotTraceWithStd.py +++ b/HPOlib/Plotting/plotTraceWithStd.py @@ -27,6 +27,8 @@ import matplotlib.gridspec import numpy as np +from HPOlib.Plotting import plot_util + __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" @@ -148,60 +150,33 @@ def main(): optimum = args.optimum title = args.title - sys.stdout.write("\nFound " + str(len(unknown)) + " arguments...") - name_list = list() + sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n") + trial_list = list() - len_list = list() - now_data = False - for i in range(len(unknown)): - if not ".pkl" in unknown[i] and now_data: - raise ValueError("You need at least on .pkl file per Experiment, %s has none" % name_list[-1]) - elif not ".pkl" in unknown[i] and not now_data: - print "Adding", unknown[i] - name_list.append([unknown[i], 0]) - trial_list.append(list()) - len_list.append(0) - now_data = True - continue - else: - now_data = False - name_list[-1][1] += 1 - - result_file = unknown[i] - fh = open(result_file, "r") - trials = cPickle.load(fh) - fh.close() - - trace = list() - currentbest = np.nanmax(np.array([trial["result"] for trial in trials['trials']])) - - for result in [trial["result"] for trial in trials['trials']]: - if result < currentbest: - currentbest = result - trace.append(currentbest) - if len(trace) > len_list[-1]: - len_list[-1] = len(trace) - trial_list[-1].append(np.array(trace)) - - if now_data: - raise ValueError("You need at least on .pkl file per Experiment, %s has none" % name_list[-1]) - - if args.autofill: - assert(len(trial_list) == len(len_list)) - for i in range(len(trial_list)): - for t in range(len(trial_list[i])): - if len(trial_list[i][t]) < len_list[i]: - diff = len_list[i] - len(trial_list[i][t]) - # noinspection PyUnusedLocal - trial_list[i][t] = np.append(trial_list[i][t], [trial_list[i][t][-1] for x in range(diff)]) - assert len(trial_list[i][t]) == len_list[i], \ - "%s, %s" % (len(trial_list[i][t]), len_list[i]) - else: - for i in range(len(trial_list)): - len_trials = len(trial_list[i][0]) - for t in trial_list[i]: - if len(t) != len_trials: - raise ValueError("(%s != %s), Traces do not have the same length, please use -a" % (str(len(t)), str(len_trials))) + + pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) + + for i in range(len(pkl_list)): + trial_list.append(list()) + for pkl in pkl_list[i]: + fh = open(pkl, "r") + trials = cPickle.load(fh) + fh.close() + + trace = plot_util.extract_trajectory(trials) + trial_list[-1].append(np.array(trace)) + + for i in range(len(trial_list)): + max_len = max([len(ls) for ls in trial_list[i]]) + for t in range(len(trial_list[i])): + if len(trial_list[i][t]) < max_len and args.autofill: + diff = max_len - len(trial_list[i][t]) + # noinspection PyUnusedLocal + trial_list[i][t] = np.append(trial_list[i][t], [trial_list[i][t][-1] for x in range(diff)]) + elif len(trial_list[i][t]) < max_len and not args.autofill: + raise ValueError("(%s != %s), Traces do not have the same length, please use -a" % + (str(max_len), str(len(trial_list[i][t])))) + plot_optimization_trace(trial_list, name_list, optimum, title=title, log=not args.log, save=args.save, y_min=args.min, y_max=args.max, scale_std=args.scale) diff --git a/HPOlib/Plotting/plot_util.py b/HPOlib/Plotting/plot_util.py new file mode 100644 index 00000000..445c89bb --- /dev/null +++ b/HPOlib/Plotting/plot_util.py @@ -0,0 +1,59 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import numpy as np + +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" + + +def get_pkl_and_name_list(argument_list): + name_list = list() + pkl_list = list() + now_data = False + for i in range(len(argument_list)): + if not ".pkl" in argument_list[i] and now_data: + raise ValueError("You need at least on .pkl file per Experiment, %s has none" % name_list[-1]) + elif not ".pkl" in argument_list[i] and not now_data: + print "Adding", argument_list[i] + name_list.append([argument_list[i], 0]) + pkl_list.append(list()) + now_data = True + continue + else: + if os.path.exists(argument_list[i]): + now_data = False + name_list[-1][1] += 1 + pkl_list[-1].append(argument_list[i]) + else: + raise ValueError("%s is not a valid file" % argument_list[i]) + if now_data: + raise ValueError("You need at least one .pkl file per Experiment, %s has none" % name_list[-1]) + return pkl_list, name_list + + +def extract_trajectory(trials): + trace = list() + currentbest = np.nanmax(np.array([trial["result"] for trial in trials['trials']])) + + for result in [trial["result"] for trial in trials['trials']]: + if result < currentbest: + currentbest = result + trace.append(currentbest) + return trace \ No newline at end of file From aa94d1df4aa50617012b3776c936be7466e1c5f4 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 25 Feb 2014 10:25:57 +0100 Subject: [PATCH 040/139] REFACTOR: #16, #1: refactor TimeTrace + is now called plotOptimizerOverhead to clarify * uses now argparse * easier interface * plots std on logscale * uses plot_util --- HPOlib/Plotting/plotOptimizerOverhead.py | 193 +++++++++++++++++++ HPOlib/Plotting/plotTimeTrace.py | 224 ----------------------- HPOlib/Plotting/plotTraceWithStd.py | 2 +- 3 files changed, 194 insertions(+), 225 deletions(-) create mode 100644 HPOlib/Plotting/plotOptimizerOverhead.py delete mode 100644 HPOlib/Plotting/plotTimeTrace.py diff --git a/HPOlib/Plotting/plotOptimizerOverhead.py b/HPOlib/Plotting/plotOptimizerOverhead.py new file mode 100644 index 00000000..f02192a8 --- /dev/null +++ b/HPOlib/Plotting/plotOptimizerOverhead.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python + +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +from argparse import ArgumentParser + +import cPickle +import itertools +import sys + +import matplotlib.pyplot as plt +import matplotlib.gridspec +import numpy as np + +from HPOlib.Plotting import plot_util + +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" + + +def plot_time_trace(time_dict, name_list, title="", log=True, save="", y_max=0, y_min=0): + colors = itertools.cycle(['b', 'g', 'r', 'k']) + markers = itertools.cycle(['o', 's', 'x', '^']) + linestyles = itertools.cycle(['-']) + + size = 5 + ratio = 5 + gs = matplotlib.gridspec.GridSpec(ratio, 1) + fig = plt.figure(1, dpi=100) + fig.suptitle(title, fontsize=16) + ax1 = plt.subplot(gs[0:ratio, :]) + ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5) + min_val = sys.maxint + max_val = -sys.maxint + max_trials = 0 + + trial_list_means = list() + trial_list_std = list() + num_runs_list = list() + + # Get mean and std for all times and optimizers + for entry in name_list: + k = entry[0] + trial_list_std.append(np.std(np.array(time_dict[k]), axis=0)) + if log: + trial_list_means.append(np.log10(np.mean(np.array(time_dict[k]), axis=0))) + else: + trial_list_means.append(np.mean(np.array(time_dict[k]), axis=0)) + num_runs_list.append(len(time_dict[k])) + + for k in range(len(name_list)): + # Plot mean and std for optimizer duration + c = colors.next() + m = markers.next() + x = range(len(trial_list_means[k])) + l = linestyles.next() + ax1.fill_between(x, trial_list_means[k] - trial_list_std[k], + trial_list_means[k] + trial_list_std[k], + facecolor=c, alpha=0.3, edgecolor=c) + ax1.plot(x, trial_list_means[k], color=c, linewidth=size, label=name_list[k][0], linestyle=l, marker=m) + # Plot number of func evals for this experiment + + if min(trial_list_means[k] - trial_list_std[k]) < min_val: + min_val = min(trial_list_means[k] - trial_list_std[k]) + if max(trial_list_means[k] + trial_list_std[k]) > max_val: + max_val = max(trial_list_means[k] + trial_list_std[k]) + if len(trial_list_means[k]) > max_trials: + max_trials = len(trial_list_means[k]) + + # Descript and label the stuff + fig.suptitle(title, fontsize=16) + leg = ax1.legend(loc='best', fancybox=True) + leg.get_frame().set_alpha(0.5) + ax1.set_ylabel("Optimizer time in [sec]") + if y_max == y_min: + ax1.set_ylim([min_val-2, max_val+2]) + else: + ax1.set_ylim([y_min, y_max]) + ax1.set_xlim([0, max_trials]) + + plt.tight_layout() + plt.subplots_adjust(top=0.85) + if save != "": + plt.savefig(save, dpi=100, facecolor='w', edgecolor='w', + orientation='portrait', papertype=None, format=None, + transparent=False, bbox_inches="tight", pad_inches=0.1) + else: + plt.show() + + +def main(): + prog = "python plotOptimizerOverhead.py WhatIsThis [WhatIsThis ]" + description = "Plot a Trace with std for multiple experiments" + + parser = ArgumentParser(description=description, prog=prog) + + # General Options + parser.add_argument("-l", "--nolog", action="store_true", dest="log", + default=False, help="Do NOT plot on log scale") + parser.add_argument("--max", type=float, dest="max", + default=0, help="Maximum of the plot") + parser.add_argument("--min", type=float, dest="min", + default=0, help="Minimum of the plot") + parser.add_argument("-s", "--save", dest="save", + default="", help="Where to save plot instead of showing it?") + parser.add_argument("-t", "--title", dest="title", + default="", help="Choose a supertitle for the plot") + + # Options which are available only for this plot + parser.add_argument("-a", "--autofill", action="store_true", dest="autofill", + default=False, help="Fill trace automatically") + + args, unknown = parser.parse_known_args() + + sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n") + + pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) + + times_dict = dict() + for exp in range(len(name_list)): + times_dict[name_list[exp][0]] = list() + for pkl in pkl_list[exp]: + fh = open(pkl, "r") + trials = cPickle.load(fh) + fh.close() + + # Get optimizer duration times + time_list = list() + # First duration + time_list.append(trials["cv_starttime"][0] - trials["starttime"][0]) + time_idx = 0 + for i in range(len(trials["cv_starttime"][1:])): + # Is there a next restored run? + # if yes, does next cvstart belong to a restored run? + if len(trials["endtime"]) > time_idx and \ + trials["cv_starttime"][i+1] > trials["endtime"][time_idx]: + # Check whether run crashed/terminated during cv + # Equals means that the run crashed + if trials["cv_endtime"][i] < trials["endtime"][time_idx]: + # No .. everything is fine + time_list.append((trials["endtime"][time_idx] - trials["cv_endtime"][i])) + time_list.append((trials["cv_starttime"][i + 1] - trials["starttime"][time_idx+1])) + elif trials["endtime"][time_idx] == trials["cv_endtime"][i]: + # Yes, but BBoM managed to set an endtime + pass + else: + # Yes ... trouble + print "Help" + print trials["endtime"][time_idx] + print trials["cv_endtime"][i] + time_idx += 1 + # everything ... + else: + time_list.append(trials["cv_starttime"][i + 1] - trials["cv_endtime"][i]) + times_dict[name_list[exp][0]].append(time_list) + + for key in times_dict.keys(): + max_len = max([len(ls) for ls in times_dict[key]]) + for t in range(len(times_dict[key])): + if len(times_dict[key][t]) < max_len and args.autofill: + diff = max_len - len(times_dict[key][t]) + # noinspection PyUnusedLocal + times_dict[key][t] = np.append(times_dict[key][t], [times_dict[key][t][-1] for x in range(diff)]) + elif len(times_dict[key][t]) < max_len and not args.autofill: + raise ValueError("(%s != %s), Traces do not have the same length, please use -a" % + (str(max_len), str(len(times_dict[key][t])))) + title = args.title + + plot_time_trace(times_dict, name_list, title=title, log=not args.log, save=args.save, + y_min=args.min, y_max=args.max) + + if args.save != "": + sys.stdout.write("Saved plot to " + args.save + "\n") + else: + sys.stdout.write("..Done\n") + +if __name__ == "__main__": + main() diff --git a/HPOlib/Plotting/plotTimeTrace.py b/HPOlib/Plotting/plotTimeTrace.py deleted file mode 100644 index 41e9d87f..00000000 --- a/HPOlib/Plotting/plotTimeTrace.py +++ /dev/null @@ -1,224 +0,0 @@ -## -# wrapping: A program making it easy to use hyperparameter -# optimization software. -# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -#!/usr/bin/env python - -import cPickle -import itertools -import optparse -import os -import sys - -import matplotlib.pyplot as plt -import matplotlib.gridspec as gridspec -import numpy as np - -__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] -__contact__ = "automl.org" - - -def plot_time_trace(timeDict, nameList, title = "", log=True, save="", verbose=False, \ - Ymax=0, Ymin=0): - colors = itertools.cycle(['b', 'g', 'r', 'k']) - markers = itertools.cycle(['o', 's', 'x', '^']) - linestyles = itertools.cycle(['-']) - size = 1 - labels = list() - ratio = 5 - gs = gridspec.GridSpec(ratio,1) - fig = plt.figure(figsize=(1,1), dpi=100) - - stdDict = dict() - meanDict = dict() - numDict = dict() - minYMean = sys.maxint - maxYMean = 0 - maxYNum = 0 - maxXNum = 0 - - # Get mean and std for all times and optimizers - for k in nameList: - stdDict[k] = list() - meanDict[k] = list() - numDict[k] = list() - # iterate over all funEvals - for i in range(max([len(i) for i in timeDict[k]])): - timeList = list() - # check for each #funEval if it exists for this optimizer - for run in range(len(timeDict[k])): - if len(timeDict[k][run]) > i: - timeList.append(timeDict[k][run][i]) - # calc mean and std - stdDict[k].append(np.std(np.array(timeList))) - meanDict[k].append(np.mean(np.array(timeList))) - numDict[k].append(len(timeList)) - if min(meanDict[k]) < minYMean: - minYMean = min(meanDict[k]) - if max(meanDict[k]) > maxYMean: - maxYMean = max(meanDict[k]) - if max(numDict[k]) > maxYNum: - maxYNum = max(numDict[k]) - if len(numDict[k]) > maxXNum: - maxXNum = len(numDict[k]) - - # Get the handles - if verbose: - ax1 = plt.subplot(gs[0:ratio-1, :]) - - ax2 = plt.subplot(gs[ratio-1, :], sharex=ax1) - ax2.set_xlabel("#Function evaluations") - ax2.set_ylabel("#Runs with #Function evaluations") - ax2.set_ylim([0, maxYNum+1]) - ax2.set_xlim([0, maxXNum]) - ax2.grid(True) - fig.subplots_adjust(hspace=0.3) - else: - ax1 = plt.subplot(gs[0:ratio, :]) - ax1.set_xlabel("#Function evaluations") - ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5) - - # Maybe plot on logscale - if log and minYMean >= 0: - ax1.set_yscale('log') - else: - sys.stdout.write("Plotting on logscale not possible or desired (minVal: %s)\n" % - (minYMean,)) - log = False - - # Plot the stuff - leg = list() - for k in nameList: - # Plot mean and std for optimizer duration - c = colors.next() - m = markers.next() - x = range(len(meanDict[k])) - l = linestyles.next() - if not log: - stdUp = np.array(meanDict[k])+np.array(stdDict[k]) - stdDown = np.array(meanDict[k])-np.array(stdDict[k]) - ax1.fill_between(x, meanDict[k], stdUp, \ - facecolor=c, alpha=0.3, edgecolor=c) - ax1.fill_between(x, meanDict[k], stdDown, \ - facecolor=c, alpha=0.3, edgecolor=c) - else: - sys.stdout.write("INFO: Plotting std on a logscale makes no sense\n") - ax1.plot(x, meanDict[k], color=c, linewidth=size, label=k, linestyle=l, marker=m) - # Plot number of func evals for this experiment - x = range(len(numDict[k])) - if verbose: - ax2.plot(x, numDict[k], color=c) - - # Descript and label the stuff - fig.suptitle(title, fontsize=16) - leg = ax1.legend(loc='best', fancybox=True) - leg.get_frame().set_alpha(0.5) - ax1.set_ylabel("Optimizer time in [sec]") - if Ymax == Ymin: - ax1.set_ylim([minYMean-2, maxYMean+2]) - else: - ax1.set_ylim([Ymin, Ymax]) - ax1.set_xlim([0, maxXNum]) - - if save != "": - fig.set_size_inches(10,10) - plt.savefig(save, dpi=100, facecolor='w', edgecolor='w', - orientation='portrait', papertype=None, format=None, - transparent=False, bbox_inches="tight", pad_inches=0.1) - else: - plt.show() - -def main(): - parser = optparse.OptionParser() - - # Options which are only available for this plot - parser.add_option("-v", "--verbose", dest="verbose", action="store_true", \ - default=False, help="Add a plot, which shows #experiments") - - # General Options - parser.add_option("-l", "--nolog", dest="log", action="store_true", \ - default=False, help="Do NOT plot on log scale") - parser.add_option("--max", dest="max", action="store", default=0, - type=float, help = "Maximum of the plot") - parser.add_option("--min", dest="min", action="store", default=0, - type=float, help="Minimum of the plot") - parser.add_option("-s", "--save", dest="save", default="", \ - help="Where to save plot instead of showing it?") - parser.add_option("-t", "--title", dest="title", default="", \ - help="Choose a supertitle for the plot") - - - (opts, args) = parser.parse_args() - - sys.stdout.write("\nFound " + str(len(args)) + " arguments...") - timeDict = dict() - nameList = list() - curOpt = "" - for i in range(len(args)): - if not ".pkl" in args[i]: - print "Adding", args[i] - timeDict[args[i]] = list() - nameList.append(args[i]) - curOpt = args[i] - continue - - result_file = args[i] - fh = open(result_file, "r") - trials = cPickle.load(fh) - fh.close() - - # Get optimizer duration times - timeList = list() - # First duration - timeList.append(trials["cv_starttime"][0] - trials["starttime"][0]) - time_idx = 0 - for i in range(len(trials["cv_starttime"][1:])): - # Is there a next restored run? - # if yes, does next cvstart belong to a restored run? - if len(trials["endtime"]) > time_idx and \ - trials["cv_starttime"][i+1] > trials["endtime"][time_idx]: - # Check whether run crashed/terminated during cv - # Equals means that the run crashed - if trials["cv_endtime"][i] < trials["endtime"][time_idx]: - # No .. everything is fine - timeList.append((trials["endtime"][time_idx] - trials["cv_endtime"][i])) - timeList.append((trials["cv_starttime"][i + 1] - trials["starttime"][time_idx+1])) - elif trials["endtime"][time_idx] == trials["cv_endtime"][i]: - # Yes, but BBoM managed to set an endtime - pass - else: - # Yes ... trouble - print "Help" - print trials["endtime"][time_idx] - print trials["cv_endtime"][i] - time_idx += 1 - # everything ... - else: - timeList.append(trials["cv_starttime"][i + 1] - trials["cv_endtime"][i]) - timeDict[curOpt].append(timeList) - - title = opts.title - - plot_time_trace(timeDict, nameList, title=title, log=not opts.log, save=opts.save, \ - verbose=opts.verbose, Ymin=opts.min, Ymax=opts.max) - if opts.save != "": - sys.stdout.write("Saved plot to " + opts.save + "\n") - else: - sys.stdout.write("..Done\n") - -if __name__ == "__main__": - main() diff --git a/HPOlib/Plotting/plotTraceWithStd.py b/HPOlib/Plotting/plotTraceWithStd.py index a91f1947..2e5acee4 100644 --- a/HPOlib/Plotting/plotTraceWithStd.py +++ b/HPOlib/Plotting/plotTraceWithStd.py @@ -118,7 +118,7 @@ def plot_optimization_trace(trial_list, name_list, optimum=0, title="", def main(): - prog = "%proc WhatIsThis [WhatIsThis ]" + prog = "python plotTraceWithStd.py WhatIsThis [WhatIsThis ]" description = "Plot a Trace with std for multiple experiments" parser = ArgumentParser(description=description, prog=prog) From 0fb24172a6ed7f37782eaab95a5d381e27b27c72 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 25 Feb 2014 10:27:57 +0100 Subject: [PATCH 041/139] FIX: imports --- HPOlib/Plotting/results.sh | 28 ++++++++++++++-------------- HPOlib/optimizers/tpe/tpecall.py | 1 + 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/HPOlib/Plotting/results.sh b/HPOlib/Plotting/results.sh index 7cef9f52..4d6db764 100644 --- a/HPOlib/Plotting/results.sh +++ b/HPOlib/Plotting/results.sh @@ -37,7 +37,7 @@ if [ -a "${directory}" ] fi -directory=`ls | grep "^smac-v2_06_02-partial38\>"` +directory=`ls | grep "^partial_smac\>"` if [ -a "${directory}" ] then @@ -46,16 +46,16 @@ then printf "%5s | %4s(%5s) | %5s | %10s | %10s\n" "Seed" "#run" "crash" "#iter" "Performance" "Test-Perf" for i in `seq 1000 1000 10000` do - directory=`ls | grep "^smac-v2_06_02-partial38_${i}_"` - if [ -f "${directory}/smac-v2_06_02-partial38.out" ] + directory=`ls | grep "^partial_smac_${i}_"` + if [ -f "${directory}/partial_smac.out" ] then - it=`cat ${directory}/smac*.out | grep "Model/Iteration used:" | tail -1` + it=`cat ${directory}/partial_*.out | grep "Model/Iteration used:" | tail -1` it=`echo $it | cut -d' ' -f3` - per=`cat ${directory}/smac*.out | grep "Performance of the Incumbent:" | tail -1` + per=`cat ${directory}/partial_*.out | grep "Performance of the Incumbent:" | tail -1` per=`echo $per | cut -d' ' -f5` - num=`cat ${directory}/smac*.out | grep "Algorithm Runs used:" | tail -1` + num=`cat ${directory}/partial_*.out | grep "Algorithm Runs used:" | tail -1` num=`echo $num | cut -d' ' -f4` numC=`ls ${directory}/ | grep 'instance.out$' | wc -l` @@ -83,12 +83,12 @@ then printf "%5s | %4s(%5s) | %10s | %10s\n" "Seed" "#run" "crash" "Performance" "Test-Perf" for i in `seq 1000 1000 10000` do - directory=`ls | grep "^tpe_${i}_"` - if [ -a "${directory}/tpe.out" ] + directory=`ls | grep "^hyperopt_august2013_mod_${i}_"` + if [ -a "${directory}/hyperopt_august2013_mod.out" ] then - num=`cat ${directory}/tpe.out | grep "Result:" | wc -l` + num=`cat ${directory}/hyperopt_august2013_mod.out | grep "Result:" | wc -l` - per=`cat ${directory}/tpe.out | grep "Result:" | sort -r | tail -1` + per=`cat ${directory}/hyperopt_august2013_mod.out | grep "Result:" | sort -r | tail -1` per=`echo $per | cut -d' ' -f2` numC=`ls ${directory}/ | grep 'instance.out$' | wc -l` @@ -115,12 +115,12 @@ if [ -a "${directory}" ] printf "%5s | %4s(%5s) | %10s | %10s\n" "Seed" "#run" "crash" "Performance" "Test-Perf" for i in `seq 1000 1000 10000` do - directory=`ls | grep "^spearmint_${i}_"` - if [ -a "${directory}/spearmint.out" ] + directory=`ls | grep "^spearmint_april2013_mod_${i}_"` + if [ -a "${directory}/spearmint_april2013_mod.out" ] then - num=`cat ${directory}/spearmint.out | grep " pending " | tail -1` + num=`cat ${directory}/spearmint_april2013_mod.out | grep " pending " | tail -1` num=`echo $num | cut -d' ' -f5` - per=`cat ${directory}/spearmint.out | grep "best:" | tail -1` + per=`cat ${directory}/spearmint_april2013_mod.out | grep "best:" | tail -1` per=`echo $per | cut -d' ' -f3` numC=`ls ${directory}/ | grep 'instance.out$' | wc -l` diff --git a/HPOlib/optimizers/tpe/tpecall.py b/HPOlib/optimizers/tpe/tpecall.py index 5adf057f..7eea63a2 100644 --- a/HPOlib/optimizers/tpe/tpecall.py +++ b/HPOlib/optimizers/tpe/tpecall.py @@ -24,6 +24,7 @@ from importlib import import_module import logging import os +import sys import hyperopt From 403dd49236e2f7ec8920b3ff1664f88b906d461b Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 26 Feb 2014 18:05:01 +0100 Subject: [PATCH 042/139] ADD: #1: add plotTraceWithStd_perTime.py --- HPOlib/Plotting/plotOptimizerOverhead.py | 5 ++++- ...lotTraceWithStd.py => plotTraceWithStd_perEval.py} | 0 HPOlib/Plotting/plot_param.py | 4 ++-- HPOlib/Plotting/plot_util.py | 11 ++++++++++- 4 files changed, 16 insertions(+), 4 deletions(-) rename HPOlib/Plotting/{plotTraceWithStd.py => plotTraceWithStd_perEval.py} (100%) diff --git a/HPOlib/Plotting/plotOptimizerOverhead.py b/HPOlib/Plotting/plotOptimizerOverhead.py index f02192a8..a22df97d 100644 --- a/HPOlib/Plotting/plotOptimizerOverhead.py +++ b/HPOlib/Plotting/plotOptimizerOverhead.py @@ -86,7 +86,10 @@ def plot_time_trace(time_dict, name_list, title="", log=True, save="", y_max=0, fig.suptitle(title, fontsize=16) leg = ax1.legend(loc='best', fancybox=True) leg.get_frame().set_alpha(0.5) - ax1.set_ylabel("Optimizer time in [sec]") + if log: + ax1.set_ylabel("log10(Optimizer time in [sec])") + else: + ax1.set_ylabel("Optimizer time in [sec]") if y_max == y_min: ax1.set_ylim([min_val-2, max_val+2]) else: diff --git a/HPOlib/Plotting/plotTraceWithStd.py b/HPOlib/Plotting/plotTraceWithStd_perEval.py similarity index 100% rename from HPOlib/Plotting/plotTraceWithStd.py rename to HPOlib/Plotting/plotTraceWithStd_perEval.py diff --git a/HPOlib/Plotting/plot_param.py b/HPOlib/Plotting/plot_param.py index f915a913..5d1cfb5a 100644 --- a/HPOlib/Plotting/plot_param.py +++ b/HPOlib/Plotting/plot_param.py @@ -78,9 +78,9 @@ def plot_params(value_list, result_list, name, save="", title="", jitter=0): for idx in range(len(result_list)): # noinspection PyArgumentList - result_list[idx] += (np.random.rand(1) - 0.5) * jitter + result_list[idx] += float((np.random.rand(1) - 0.5) * jitter) # noinspection PyArgumentList - value_list[idx] += (np.random.rand(1) - 0.5) * jitter + value_list[idx] += float((np.random.rand(1) - 0.5) * jitter) # Plot ax.scatter(value_list, result_list, facecolor=color, edgecolor=color, marker=marker, s=size*50) diff --git a/HPOlib/Plotting/plot_util.py b/HPOlib/Plotting/plot_util.py index 445c89bb..52f32d82 100644 --- a/HPOlib/Plotting/plot_util.py +++ b/HPOlib/Plotting/plot_util.py @@ -56,4 +56,13 @@ def extract_trajectory(trials): if result < currentbest: currentbest = result trace.append(currentbest) - return trace \ No newline at end of file + return trace + + +def extract_runtime_timestamps(trials): + # return a list like (20, 53, 101, 200) + time_list = list() + time_list.append(0) + for trial in trials["trials"]: + time_list.append(np.sum(trial["instance_durations"]) + time_list[-1]) + return time_list \ No newline at end of file From d24ffc5d394c56cd151067a0676104428972eaa2 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 26 Feb 2014 18:06:33 +0100 Subject: [PATCH 043/139] FIX: forgot file --- HPOlib/Plotting/plotTraceWithStd_perTime.py | 257 ++++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 HPOlib/Plotting/plotTraceWithStd_perTime.py diff --git a/HPOlib/Plotting/plotTraceWithStd_perTime.py b/HPOlib/Plotting/plotTraceWithStd_perTime.py new file mode 100644 index 00000000..638af260 --- /dev/null +++ b/HPOlib/Plotting/plotTraceWithStd_perTime.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python + +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from argparse import ArgumentParser +import cPickle +import itertools +import sys + +from matplotlib.pyplot import tight_layout, figure, subplots_adjust, subplot, savefig, show +import matplotlib.gridspec +import numpy as np + +from HPOlib.Plotting import plot_util + +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" + + +def plot_optimization_trace(trial_list, name_list, times_list, optimum=0, title="", + log=True, save="", y_max=0, y_min=0, scale_std=1): + # markers = itertools.cycle(['o', 's', 'x', '^']) + colors = itertools.cycle(['b', 'g', 'r', 'k']) + linestyles = itertools.cycle(['-']) + size = 1 + + ratio = 5 + gs = matplotlib.gridspec.GridSpec(ratio, 1) + fig = figure(1, dpi=100) + fig.suptitle(title, fontsize=16) + ax1 = subplot(gs[0:ratio, :]) + ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5) + min_val = sys.maxint + max_val = -sys.maxint + max_trials = 0 + + trial_list_means = list() + trial_list_std = list() + + # One trialList represents all runs from one optimizer + for i in range(len(trial_list)): + if log: + trial_list_means.append(np.log10(np.mean(np.array(trial_list[i]), axis=0))) + else: + trial_list_means.append(np.mean(np.array(trial_list[i]), axis=0)) + trial_list_std.append(np.std(np.array(trial_list[i]), axis=0)*scale_std) + times_list[i] = np.array(times_list[i]) + + fig.suptitle(title, fontsize=16) + + # Plot the average error and std + for i in range(len(trial_list_means)): + x = times_list[i] + y = trial_list_means[i] - optimum + # m = markers.next() + c = colors.next() + l = linestyles.next() + std_up = y + trial_list_std[i] + std_down = y - trial_list_std[i] + + ax1.fill_between(x, std_down, std_up, + facecolor=c, alpha=0.3, edgecolor=c) + ax1.plot(x, y, color=c, linewidth=size*2, + label=name_list[i][0] + "(" + str(len(trial_list[i])) + ")", + linestyle=l, marker="") + if min(std_down) < min_val: + min_val = min(std_down) + if max(y + std_up) > max_val: + max_val = max(std_up) + if max(times_list[i]) > max_trials: + max_trials = max(times_list[i]) + + # Maybe plot on logscale + if scale_std != 1: + ylabel = ", %s * std" % scale_std + else: + ylabel = "" + + if log: + ax1.set_ylabel("log10(Minfunction value)" + ylabel) + else: + ax1.set_ylabel("Minfunction value" + ylabel) + + # Descript and label the stuff + leg = ax1.legend(loc='best', fancybox=True) + leg.get_frame().set_alpha(0.5) + ax1.set_xlabel("duration") + + if y_max == y_min: + # Set axes limit + ax1.set_ylim([min_val-0.1*abs((max_val-min_val)), max_val+0.1*abs((max_val-min_val))]) + else: + ax1.set_ylim([y_min, y_max]) + ax1.set_xlim([0, max_trials]) + + tight_layout() + subplots_adjust(top=0.85) + if save != "": + savefig(save, dpi=100, facecolor='w', edgecolor='w', + orientation='portrait', papertype=None, format=None, + transparent=False, bbox_inches="tight", pad_inches=0.1) + else: + show() + + +def fill_trajectories(trace_list, times_list): + """ Each trajectory need to has the exact same number of entries and timestamps""" + # We need to define the max value = what is measured before the first evaluation + max_value = np.max([np.max(ls) for ls in trace_list]) + + number_exp = len(trace_list) + new_trajectories = list() + new_times = list() + for i in range(number_exp): + new_trajectories.append(list()) + new_times.append(list()) + # noinspection PyUnusedLocal + counter = [1 for i in range(number_exp)] + finish = False + + # We need to insert the max values + for i in range(number_exp): + trace_list[i].insert(0, max_value) + trace_list[i].append(0) + times_list[i].insert(0, 0) + times_list[i].append(sys.maxint) + + # Add all possible time values + while not finish: + min_idx = np.argmin([times_list[idx][counter[idx]] for idx in range(number_exp)]) + counter[min_idx] += 1 + for idx in range(number_exp): + new_times[idx].append(times_list[min_idx][counter[min_idx] - 1]) + new_trajectories[idx].append(trace_list[idx][counter[idx] - 1]) + # Check if we're finished + for i in range(number_exp): + finish = True + if counter[i] < len(trace_list[i]) - 1: + finish = False + break + + times = new_times + trajectories = new_trajectories + tmp_times = list() + + # Sanitize lists and delete double entries + for i in range(number_exp): + tmp_times = list() + tmp_traj = list() + for t in range(len(times[i]) - 1): + if times[i][t+1] != times[i][t] and not np.isnan(times[i][t]): + tmp_times.append(times[i][t]) + tmp_traj.append(trajectories[i][t]) + tmp_times.append(times[i][-1]) + tmp_traj.append(trajectories[i][-1]) + times[i] = tmp_times + trajectories[i] = tmp_traj + + # We need only one list for all times + times = tmp_times + + return trajectories, times + + +def main(): + prog = "python plotTraceWithStd.py WhatIsThis [WhatIsThis ]" + description = "Plot a Trace with std for multiple experiments" + + parser = ArgumentParser(description=description, prog=prog) + + # Options for specific benchmarks + parser.add_argument("-o", "--optimum", type=float, dest="optimum", + default=0, help="If not set, the optimum is supposed to be zero") + + # Options which are available only for this plot + parser.add_argument("-a", "--autofill", action="store_true", dest="autofill", + default=False, help="Fill trace automatically") + parser.add_argument("-c", "--scale", type=float, dest="scale", + default=1, help="Multiply std to get a nicer plot") + # General Options + parser.add_argument("-l", "--nolog", action="store_true", dest="log", + default=False, help="Do NOT plot on log scale") + parser.add_argument("--max", dest="max", type=float, + default=0, help="Maximum of the plot") + parser.add_argument("--min", dest="min", type=float, + default=0, help="Minimum of the plot") + parser.add_argument("-s", "--save", dest="save", + default="", help="Where to save plot instead of showing it?") + parser.add_argument("-t", "--title", dest="title", + default="", help="Optional supertitle for plot") + + args, unknown = parser.parse_known_args() + + sys.stdout.write("Generate plot for ") + optimum = args.optimum + title = args.title + + sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n") + + trial_list = list() + times_list = list() + + pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) + + for i in range(len(pkl_list)): + tmp_trial_list = list() + tmp_times_list = list() + for pkl in pkl_list[i]: + fh = open(pkl, "r") + trials = cPickle.load(fh) + fh.close() + + trace = plot_util.extract_trajectory(trials) + times = plot_util.extract_runtime_timestamps(trials) + tmp_times_list.append(times) + tmp_trial_list.append(trace) + # We feed this function with two lists of lists and get one list of lists and one list + tmp_trial_list, tmp_times_list = fill_trajectories(tmp_trial_list, tmp_times_list) + trial_list.append(tmp_trial_list) + times_list.append(tmp_times_list) + + for i in range(len(trial_list)): + max_len = max([len(ls) for ls in trial_list[i]]) + for t in range(len(trial_list[i])): + if len(trial_list[i][t]) < max_len and args.autofill: + diff = max_len - len(trial_list[i][t]) + # noinspection PyUnusedLocal + trial_list[i][t] = np.append(trial_list[i][t], [trial_list[i][t][-1] for x in range(diff)]) + elif len(trial_list[i][t]) < max_len and not args.autofill: + raise ValueError("(%s != %s), Traces do not have the same length, please use -a" % + (str(max_len), str(len(trial_list[i][t])))) + + plot_optimization_trace(trial_list, name_list, times_list, optimum, title=title, log=not args.log, + save=args.save, y_min=args.min, y_max=args.max, scale_std=args.scale) + if args.save != "": + sys.stdout.write("Saved plot to " + args.save + "\n") + else: + sys.stdout.write("..Done\n") + +if __name__ == "__main__": + main() From f8a6b4a4312488be556af264fd3485755707dcfc Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Fri, 28 Feb 2014 10:11:12 +0100 Subject: [PATCH 044/139] REFACTOR: #16, #1: refactor plotTrace.py * use now argparse * clarify interface --- HPOlib/Plotting/plotTrace.py | 201 ++++++++++++++++++----------------- HPOlib/Plotting/plot_util.py | 5 + 2 files changed, 108 insertions(+), 98 deletions(-) diff --git a/HPOlib/Plotting/plotTrace.py b/HPOlib/Plotting/plotTrace.py index 0542a02e..3517e716 100644 --- a/HPOlib/Plotting/plotTrace.py +++ b/HPOlib/Plotting/plotTrace.py @@ -18,48 +18,47 @@ #!/usr/bin/env python -import cPickle +from argparse import ArgumentParser import itertools -import optparse -import os import sys +import cPickle -import matplotlib.pyplot as plt -import matplotlib.gridspec as gridspec +from matplotlib.pyplot import tight_layout, figure, subplots_adjust, subplot, savefig, show, yscale +import matplotlib.gridspec import numpy as np import scipy.stats as sc +from HPOlib.Plotting import plot_util + __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" -def plot_optimization_trace(opts, trialList, nameList, optimum = 0, title="", Ymin=0, Ymax=0): +def plot_optimization_trace(trial_list, name_list, optimum=0, title="", log=False, + save="", y_min=0, y_max=0): markers = itertools.cycle(['o', 's', '^', 'x']) colors = itertools.cycle(['b', 'g', 'r', 'k']) linestyles = itertools.cycle(['-']) size = 1 - labels = list() - plotH = list() - - # Get handles - ratio=5 - gs = gridspec.GridSpec(ratio,1) - fig = plt.figure(1, dpi=100) - fig.suptitle(title) - ax = plt.subplot(gs[0:ratio, :]) + + # get handles + ratio = 5 + gs = matplotlib.gridspec.GridSpec(ratio, 1) + fig = figure(1, dpi=100) + fig.suptitle(title, fontsize=16) + ax = subplot(gs[0:ratio, :]) ax.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5) - # Get values - minVal = sys.maxint - maxVal = 0 - maxTrials = 0 + min_val = sys.maxint + max_val = -sys.maxint + max_trials = 0 + # This might not do what we actually want; Ideally it would only take the # ones that where actually run according to instance_order - for i in range(len(nameList)): - x = range(len(trialList[i]['trials'])) - y = np.zeros((len(trialList[i]['trials']))) - line = np.zeros((len(trialList[i]['trials']))) - for j, trial in enumerate(trialList[i]['trials']): - inst_res = trial['instance_results'] + for i in range(len(name_list)): + x = range(len(trial_list[i])) + y = np.zeros((len(trial_list[i]))) + line = np.zeros((len(trial_list[i]))) + for j, inst_res in enumerate(trial_list[i]): if j >= len(y): break if type(inst_res) == np.ndarray and not np.isfinite(inst_res).any(): @@ -67,97 +66,103 @@ def plot_optimization_trace(opts, trialList, nameList, optimum = 0, title="", Ym elif type(inst_res) != np.ndarray and np.isnan(inst_res): inst_res = 1 tmp = sc.nanmean(np.array([inst_res, inst_res]).flat) # Black Magic - y[j] = tmp - optimum - line[j] = np.min(y[:j + 1]) + if log: + y[j] = np.log(tmp - optimum) + line[j] = np.min(y[:j + 1]) + else: + y[j] = tmp - optimum + line[j] = np.min(y[:j + 1]) + # Plot the stuff marker = markers.next() color = colors.next() l = linestyles.next() - ax.scatter(np.argmin(line), min(line), facecolor="w", edgecolor=color, \ + ax.scatter(np.argmin(line), min(line), facecolor="w", edgecolor=color, s=size*10*15, marker=marker) ax.scatter(x, y, color=color, marker=marker, s=size*15) - ax.plot(x, line, color=color, label=nameList[i], linestyle=l, linewidth=size) - labels.append(trialList[i]['experiment_name'] + " (" + str(min(y)) + ")") - if min(y) < minVal: - minVal = min(y) - if max(y) > maxVal: - maxVal = max(y) - if len(trialList[i]) > maxTrials: - maxTrials = len(trialList[i]['trials']) + ax.plot(x, line, color=color, label=name_list[i][0], linestyle=l, linewidth=size) + + if min(y) < min_val: + min_val = min(y) + if max(y) > max_val: + max_val = max(y) + if len(trial_list[i]) > max_trials: + max_trials = len(trial_list[i]) # Describe and label the stuff ax.set_xlabel("#Function evaluations") - ax.set_ylabel("Minfunction value") + if log: + ax.set_ylabel("log10(Minfunction value)") + else: + ax.set_ylabel("Minfunction value") - if Ymin == Ymax: - ax.set_ylim([minVal-0.1, maxVal+0.1]) + if y_min == y_max: + ax.set_ylim([min_val - 0.1, max_val + 0.1]) else: - ax.set_ylim([Ymin, Ymax]) - + ax.set_ylim([y_min, y_max]) + ax.set_xlim([0, max_trials]) + leg = ax.legend(loc='best', fancybox=True) leg.get_frame().set_alpha(0.5) - ax.set_xlim([0, maxTrials]) - plt.subplots_adjust(top=0.85) - if opts.log: - plt.yscale('log') - if opts.save != "": - plt.savefig(opts.save, dpi=100, facecolor='w', edgecolor='w', - orientation='portrait', papertype=None, format=None, - transparent=False, bbox_inches="tight", pad_inches=0.1) + + tight_layout() + subplots_adjust(top=0.85) + + if save != "": + savefig(save, dpi=100, facecolor='w', edgecolor='w', + orientation='portrait', papertype=None, format=None, + transparent=False, bbox_inches="tight", pad_inches=0.1) else: - plt.show() - + show() + + def main(): - parser = optparse.OptionParser() - # Options for specific functions - parser.add_option("-o", "--optimum", type = float, dest = "optimum", - default = 0, help = "Optimum") - parser.add_option("-b", "--branin", dest = "branin", action = "store_true", - default = False, help = "Automatic shift for branin function") - parser.add_option("-c", "--camel", dest="camelback", action="store_true", - default = False, help="Automatic shift for camelback function") - parser.add_option("-i", "--har3", dest = "har3", action = "store_true", - default = False, help = "Automatic shift for hartmann3 function") - parser.add_option("-j", "--har6", dest = "har6", action = "store_true", - default = False, help = "Automatic shift for hartmann6 function") - parser.add_option("-g", "--gold", dest = "gold", action = "store_true", - default = False, help = "Automatic shift for goldstein function") - + prog = "python plotTrace.py WhatIsThis [WhatIsThis ]" + description = "Plot a Trace with evaluated points wrt to performance" + + parser = ArgumentParser(description=description, prog=prog) + + # Options for specific benchmarks + parser.add_argument("-o", "--optimum", type=float, dest="optimum", + default=0, help="If not set, the optimum is supposed to be zero") # General Options - parser.add_option("-l", "--log", dest = "log", action = "store_true", - default = False, help = "Plot on the log scale") - parser.add_option("--max", dest = "max", action = "store", default = 0, - type = float, help = "Maximum of the plot") - parser.add_option("--min", dest = "min", action = "store", default = 0, - type = float, help = "Minimum of the plot") - parser.add_option("-s", "--save", dest = "save", default = "", \ - help = "Where to save plot instead of showing it?") - parser.add_option("-t", "--title", dest="title", \ - default="", help="Optional supertitle for plot") - (opts, args) = parser.parse_args() - - optimum = opts.optimum - if opts.branin: optimum = 0.397887 - if opts.camelback: optimum = -1.031628453 - if opts.har3: optimum = -3.86278 - if opts.har6: optimum = -3.32237 - if opts.gold: optimum = -3 - - sys.stdout.write("Found " + str(len(args)) + " pkl file(s)\n") - trialList = list() - nameList = list() - for i in range(len(args)): - if not ".pkl" in args[i]: - print "Adding", args[i] - nameList.append(args[i]) - continue - result_file = args[i] - fh = open(result_file, "r") - trials = cPickle.load(fh) + parser.add_argument("-l", "--nolog", action="store_true", dest="log", + default=False, help="Do NOT plot on log scale") + parser.add_argument("--max", dest="max", type=float, + default=0, help="Maximum of the plot") + parser.add_argument("--min", dest="min", type=float, + default=0, help="Minimum of the plot") + parser.add_argument("-s", "--save", dest="save", + default="", help="Where to save plot instead of showing it?") + parser.add_argument("-t", "--title", dest="title", + default="", help="Optional supertitle for plot") + + args, unknown = parser.parse_known_args() + + optimum = args.optimum + + sys.stdout.write("Found " + str(len(unknown)) + " arguments\n") + + pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) + + trial_list = list() + for i in range(len(pkl_list)): + if len(pkl_list[i]) != 1: + raise ValueError("%s is more than !" % str(pkl_list)) + fh = open(pkl_list[i][0]) + trl = cPickle.load(fh) fh.close() - trialList.append(trials) + trial_list.append(plot_util.extract_trials(trl)) + sys.stdout.write("Plotting trace\n") - plot_optimization_trace(opts, trialList, nameList, optimum, title=opts.title, Ymax=opts.max, Ymin=opts.min) + plot_optimization_trace(trial_list=trial_list, name_list=name_list, optimum=optimum, + title=args.title, log=not args.log, + save=args.save, y_max=args.max, y_min=args.min) + + if args.save != "": + sys.stdout.write("Saved plot to " + args.save + "\n") + else: + sys.stdout.write("..Done\n") if __name__ == "__main__": main() diff --git a/HPOlib/Plotting/plot_util.py b/HPOlib/Plotting/plot_util.py index 52f32d82..48f7a5aa 100644 --- a/HPOlib/Plotting/plot_util.py +++ b/HPOlib/Plotting/plot_util.py @@ -59,6 +59,11 @@ def extract_trajectory(trials): return trace +def extract_trials(trials): + trl = [trial["result"] for trial in trials['trials']] + return trl + + def extract_runtime_timestamps(trials): # return a list like (20, 53, 101, 200) time_list = list() From 9f6938c237ce29ed4bdaf5f3b0e6e039ef5c867e Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 3 Mar 2014 11:00:42 +0100 Subject: [PATCH 045/139] REFACTOR: #16, #1: plotBoxWhisker.py --- HPOlib/Plotting/plotBoxWhisker.py | 184 ++++++++++++++++++------------ HPOlib/Plotting/plot_util.py | 9 +- 2 files changed, 118 insertions(+), 75 deletions(-) diff --git a/HPOlib/Plotting/plotBoxWhisker.py b/HPOlib/Plotting/plotBoxWhisker.py index 16f9a9b5..c7ce905b 100644 --- a/HPOlib/Plotting/plotBoxWhisker.py +++ b/HPOlib/Plotting/plotBoxWhisker.py @@ -1,107 +1,143 @@ #!/usr/bin/env python +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +from argparse import ArgumentParser + import cPickle -import itertools -import optparse -import os import sys -import matplotlib.gridspec as gridspec -import matplotlib.pyplot as plt +from matplotlib.gridspec import GridSpec +from matplotlib.pyplot import xticks, figure, subplot, savefig, show, tight_layout, subplots_adjust import numpy as np +from HPOlib.Plotting import plot_util + + __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __license__ = "3-clause BSD License" __contact__ = "automl.org" -def plot_boxWhisker(bestDict, title = "", save = "", Ymin=0, Ymax=0): +def plot_box_whisker(best_trials, name_list, title="", save="", y_min=0, y_max=0): ratio = 5 - gs = gridspec.GridSpec(ratio,1) - fig = plt.figure(1, dpi=100) + gs = GridSpec(ratio, 1) + fig = figure(1, dpi=100) fig.suptitle(title) - ax = plt.subplot(gs[0:ratio, :]) + ax = subplot(gs[0:ratio, :]) ax.yaxis.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5) - - labels = bestDict.keys() - bp = ax.boxplot([bestDict[i] for i in labels], 0, 'ok') + + bp = ax.boxplot(best_trials, 0, 'ok') boxlines = bp['boxes'] for line in boxlines: line.set_color('k') line.set_linewidth(2) - if Ymin != Ymax: - ax.set_ylim([Ymin, Ymax]) - # Get medians - medians = range(len(labels)) - for i in range(len(labels)): + min_y = sys.maxint + max_y = -sys.maxint + + # Get medians and limits + medians = range(len(name_list)) + for i in range(len(name_list)): med = bp['medians'][i] - medianX = [] - medianY = [] + median_x = [] + median_y = [] for j in range(2): - medianX.append(med.get_xdata()[j]) - medianY.append(med.get_ydata()[j]) - medians[i] = medianY[0] + median_x.append(med.get_xdata()[j]) + median_y.append(med.get_ydata()[j]) + medians[i] = median_y[0] + if min(best_trials[i]) < min_y: + min_y = min(best_trials[i]) + if max(best_trials[i]) > max_y: + max_y = max(best_trials[i]) print medians # Plot xticks - plt.xticks(range(1, len(bestDict.keys())+1), labels) - + xticks(range(1, len(name_list)+1), name_list) + + # Set limits + if y_max == y_min: + # Set axes limit + ax.set_ylim([min_y-0.1*abs((max_y-min_y)), max_y+0.1*abs((max_y-min_y))]) + else: + ax.set_ylim([y_min, y_max]) + max_y = y_max + min_y = y_min + # Print medians as upper labels - top = Ymax-((Ymax-Ymin)*0.05) - pos = np.arange(len(labels))+1 - upperLabels = [str(np.round(s, 5)) for s in medians] - for tick,label in zip(range(len(labels)), ax.get_xticklabels()): - k = tick % 2 - ax.text(pos[tick], top, upperLabels[tick], - horizontalalignment='center', size='x-small') - - ax.set_ylabel('Minfunction evaluation') + top = max_y-((max_y-min_y)*0.05) + pos = np.arange(len(name_list))+1 + upper_labels = [str(np.round(s, 5)) for s in medians] + upper_labels[0] = "median=[%s," % upper_labels[0] + for i in range(len(upper_labels[1:-1])): + upper_labels[i+1] = "%s," % upper_labels[i+1] + upper_labels[-1] = "%s]" % upper_labels[-1] + for tick, label in zip(range(len(name_list)), ax.get_xticklabels()): + ax.text(pos[tick], top, upper_labels[tick], + horizontalalignment='center', size='x-small') + + ax.set_ylabel('Minfunction') + + tight_layout() + subplots_adjust(top=0.85) if save != "": - plt.savefig(save, dpi=100, facecolor='w', edgecolor='w', - orientation='portrait', papertype=None, format=None, - transparent=True, bbox_inches="tight", pad_inches=0.1) + savefig(save, dpi=100, facecolor='w', edgecolor='w', + orientation='portrait', papertype=None, format=None, + transparent=False, bbox_inches="tight", pad_inches=0.1) else: - plt.show() - + show() + + def main(): - parser = optparse.OptionParser() - parser.add_option("-t", "--title", dest = "title", \ - default = "TBA", help = "Optional supertitle for plot") - parser.add_option("--max", dest = "max", action = "store", default = 0, - type = float, help = "Maximum of the plot") - parser.add_option("--min", dest = "min", action = "store", default = 0, - type = float, help = "Minimum of the plot") - parser.add_option("-s", "--save", dest = "save", default = "", \ - help = "Where to save plot instead of showing it?") - (opts, args) = parser.parse_args() - - title = "" - if opts.title == "TBA": - title = "Results for Optimizer(s)" - - sys.stdout.write("\nFound " + str(len(args)) + " arguments...") - bestDict = dict() - curOpt = "" - for i in range(len(args)): - if not ".pkl" in args[i]: - print "Adding", args[i] - bestDict[args[i]] = list() - curOpt = args[i] - continue - - result_file = args[i] - fh = open(result_file, "r") - trials = cPickle.load(fh) - fh.close() - bestDict[curOpt].append(min([trial['result'] for trial in trials['trials']])) - - plot_boxWhisker(bestDict=bestDict, title=title, save=opts.save, \ - Ymin=opts.min, Ymax=opts.max) - - if opts.save != "": - sys.stdout.write("Saving plot to " + opts.save + "\n") + prog = "python plotBoxWhisker.py WhatIsThis [WhatIsThis ]" + description = "Plot a Box whisker plot for many experiments. The box covers lower to upper quartile." + + parser = ArgumentParser(description=description, prog=prog) + + # General Options + parser.add_argument("-t", "--title", dest="title", default="", + help="Optional supertitle for plot") + parser.add_argument("--max", dest="max", default=0, + type=float, help="Maximum of the plot") + parser.add_argument("--min", dest="min", default=0, + type=float, help="Minimum of the plot") + parser.add_argument("-s", "--save", dest="save", default="", + help="Where to save plot instead of showing it?") + args, unknown = parser.parse_known_args() + + sys.stdout.write("\nFound " + str(len(unknown)) + " arguments...") + + pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) + best_trials = list() + for i in range(len(name_list)): + best_trials.append(list()) + for pkl in pkl_list[i]: + fh = open(pkl, "r") + trials = cPickle.load(fh) + fh.close() + best_trials[i].append(plot_util.get_best(trials)) + + plot_box_whisker(best_trials=best_trials, name_list=name_list, + title=args.title, save=args.save, y_min=args.min, y_max=args.max) + + if args.save != "": + sys.stdout.write("Saving plot to " + args.save + "\n") else: sys.stdout.write("..Done\n") diff --git a/HPOlib/Plotting/plot_util.py b/HPOlib/Plotting/plot_util.py index 48f7a5aa..1f1ecb36 100644 --- a/HPOlib/Plotting/plot_util.py +++ b/HPOlib/Plotting/plot_util.py @@ -18,6 +18,7 @@ import os import numpy as np +import sys __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" @@ -70,4 +71,10 @@ def extract_runtime_timestamps(trials): time_list.append(0) for trial in trials["trials"]: time_list.append(np.sum(trial["instance_durations"]) + time_list[-1]) - return time_list \ No newline at end of file + return time_list + + +def get_best(trials): + # returns the best value found in this experiment + best_value = extract_trajectory(trials)[-1] + return best_value \ No newline at end of file From b77836dbace374eeae81d02be3d6e6ea3ac50e23 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 3 Mar 2014 12:46:51 +0100 Subject: [PATCH 046/139] REFACTOR: #16, #1: statistics.py --- HPOlib/Plotting/plot_util.py | 25 +++- HPOlib/Plotting/statistics.py | 219 ++++++++++++++++------------------ 2 files changed, 126 insertions(+), 118 deletions(-) diff --git a/HPOlib/Plotting/plot_util.py b/HPOlib/Plotting/plot_util.py index 1f1ecb36..97514bd2 100644 --- a/HPOlib/Plotting/plot_util.py +++ b/HPOlib/Plotting/plot_util.py @@ -74,7 +74,26 @@ def extract_runtime_timestamps(trials): return time_list -def get_best(trials): +def get_best(trials, cut=False): # returns the best value found in this experiment - best_value = extract_trajectory(trials)[-1] - return best_value \ No newline at end of file + traj = extract_trajectory(trials) + best_value = sys.maxint + if cut and 0 < cut < len(traj): + best_value = traj[cut] + else: + best_value = traj[-1] + return best_value + + +def get_best_value_and_index(trials, cut=False): + # returns the best value and corresponding index + traj = extract_trajectory(trials) + best_value = sys.maxint + best_index = -1 + if cut and 0 < cut < len(traj): + best_value = traj[cut] + best_index = np.argmin(traj[cut]) + else: + best_value = traj[-1] + best_index = np.argmin(traj) + return best_value, best_index \ No newline at end of file diff --git a/HPOlib/Plotting/statistics.py b/HPOlib/Plotting/statistics.py index 47a05a18..5a5830c9 100644 --- a/HPOlib/Plotting/statistics.py +++ b/HPOlib/Plotting/statistics.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + ## # wrapping: A program making it easy to use hyperparameter # optimization software. @@ -16,21 +18,23 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python - +from argparse import ArgumentParser import cPickle -import optparse import sys +from distutils.version import StrictVersion import numpy -import numpy.ma as ma +from numpy.ma import asarray +import scipy from scipy import stats +from HPOlib.Plotting import plot_util + __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" -def _getBestTrial(filename, cut=None): +def _get_best_trial(filename, cut=None): try: fh = open(filename, "r") trials = cPickle.load(fh) @@ -54,45 +58,45 @@ def _getBestTrial(filename, cut=None): sys.stdout.flush() return None, None - +# TODO: Don't know whether this is longer needed +""" def _mann_whitney_u(x, y=None): - """ - Calculate the Mann-Whitney-U test. - - The Wilcoxon signed-rank test tests the null hypothesis that two related paired - samples come from the same distribution. In particular, it tests whether the - distribution of the differences x - y is symmetric about zero. - It is a non-parametric version of the paired T-test. - """ + # Calculate the Mann-Whitney-U test. + # The Wilcoxon signed-rank test tests the null hypothesis that two related paired + # samples come from the same distribution. In particular, it tests whether the + # distribution of the differences x - y is symmetric about zero. + # It is a non-parametric version of the paired T-test. + # A significance-dict for a two-tailed test with 0.05 confidence # from http://de.wikipedia.org/wiki/Wilcoxon-Mann-Whitney-Test significance_table = \ - [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2], - [0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8], - [0, 0, 0, 0, 1, 2, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 11, 12, 13, 14], - [0, 0, 0, 0, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13, 14, 15, 17, 18, 19, 20], - [0, 0, 0, 0, 0, 5, 6, 8, 10, 11, 13, 14, 16, 17, 19, 21, 22, 24, 25, 27], - [0, 0, 0, 0, 0, 0, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34], - [0, 0, 0, 0, 0, 0, 0, 13, 15, 17, 19, 22, 24, 26, 29, 31, 34, 36, 38, 41], - [0, 0, 0, 0, 0, 0, 0, 0, 17, 20, 23, 26, 28, 31, 34, 37, 39, 42, 45, 48], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 26, 29, 33, 36, 39, 42, 45, 48, 52, 55], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 33, 37, 40, 44, 47, 51, 55, 58, 62], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 41, 45, 49, 53, 57, 61, 65, 69], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 50, 54, 59, 63, 67, 72, 76], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 59, 64, 69, 74, 78, 83], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 70, 75, 80, 85, 90], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 81, 86, 92, 98], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 93, 99, 105], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 106, 112], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 119], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127]] - - U, p = stats.mannwhitneyu(x, y) - - x = ma.asarray(x).compressed().view(numpy.ndarray) - y = ma.asarray(y).compressed().view(numpy.ndarray) + [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2], + [0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8], + [0, 0, 0, 0, 1, 2, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 11, 12, 13, 14], + [0, 0, 0, 0, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13, 14, 15, 17, 18, 19, 20], + [0, 0, 0, 0, 0, 5, 6, 8, 10, 11, 13, 14, 16, 17, 19, 21, 22, 24, 25, 27], + [0, 0, 0, 0, 0, 0, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34], + [0, 0, 0, 0, 0, 0, 0, 13, 15, 17, 19, 22, 24, 26, 29, 31, 34, 36, 38, 41], + [0, 0, 0, 0, 0, 0, 0, 0, 17, 20, 23, 26, 28, 31, 34, 37, 39, 42, 45, 48], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 26, 29, 33, 36, 39, 42, 45, 48, 52, 55], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 33, 37, 40, 44, 47, 51, 55, 58, 62], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 41, 45, 49, 53, 57, 61, 65, 69], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 50, 54, 59, 63, 67, 72, 76], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 59, 64, 69, 74, 78, 83], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 70, 75, 80, 85, 90], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 81, 86, 92, 98], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 93, 99, 105], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 106, 112], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 119], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127]] + u_value, p = scipy.stats.mannwhitneyu(x, y) + + # noinspection PyNoneFunctionAssignment + x = asarray(x).compressed().view(numpy.ndarray) + y = asarray(y).compressed().view(numpy.ndarray) + n1 = len(x) n2 = len(y) print n1, n2 @@ -105,103 +109,88 @@ def _mann_whitney_u(x, y=None): return 10000, 10000 if n1 < 20 or n2 < 20: print "WARNING: scipy.stat might not be accurate, p value is %f and significance according to table is %s" % \ - (p, U <= significance_table[n1 - 1][n2 - 1]) - return U, p + (p, u_value <= significance_table[n1 - 1][n2 - 1]) + return u_value, p +""" -def _unpaired_ttest(x, y, equal_var=True): - U, p = stats.ttest_ind(x, y, equal_var=equal_var) - return U, p def main(): - usage = "python statistics.py SMAC smac_*/smac.pkl TPE tpe_*/tpe.pkl\n" - parser = optparse.OptionParser(usage) - parser.add_option("-c", "--c", type=int, dest="cut", - default=None, - help="Only consider that many evaluations") - (opts, args) = parser.parse_args() - - bestDict = dict() - idxDict = dict() + prog = "python statistics.py WhatIsThis WhatIsThis [WhatIsThis ]" + description = "Return some statistical information" + + parser = ArgumentParser(description=description, prog=prog) + + parser.add_argument("-c", "--c", dest="cut", default=False, + type=int, help="Only consider that many evaluations") + args, unknown = parser.parse_known_args() + + best_dict = dict() + idx_dict = dict() keys = list() - curOpt = "" - for i in range(len(args)): - if not ".pkl" in args[i]: - bestDict[args[i]] = list() - idxDict[args[i]] = list() - keys.append(args[i]) - curOpt = args[i] - continue - best, idx =_getBestTrial(args[i], opts.cut) - if best is None and idx is None: - continue - - # best = best * 100 - - bestDict[curOpt].append(best) - idxDict[curOpt].append(idx) + + pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) + + for i in range(len(name_list)): + keys.append(name_list[i][0]) + best_dict[name_list[i][0]] = list() + idx_dict[name_list[i][0]] = list() + for pkl in pkl_list[i]: + fh = open(pkl) + trial = cPickle.load(fh) + fh.close() + best, idx = plot_util.get_best_value_and_index(trial, args.cut) + best_dict[name_list[i][0]].append(best) + idx_dict[name_list[i][0]].append(idx) for k in keys: - sys.stdout.write("%10s: %s experiments\n" % (k, len(bestDict[k]))) - """ - sys.stdout.write("Wilcoxon Test--------------------------------------------------------\n") - for idx, k in enumerate(keys): - if len(keys) > 1: - for jdx, j in enumerate(keys[idx+1:]): - #if len(bestDict[k]) == len(bestDict[j]): - T, p = _mann_whitney_u(bestDict[k], bestDict[j]) - sys.stdout.write("%10s vs %10s" % (k,j)) - sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" % - (T, p, p*100)) - #else: - # print "WARNING: %s has not the same number of function evals than %s" % \ - # (k, j) - # continue - else: - T, p = _mann_whitney_u(bestDict[k], bestDict[k]) - sys.stdout.write("%10s vs. %10s" % (k,k)) - sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" % - (T, p, p*100)) - """ + sys.stdout.write("%10s: %s experiment(s)\n" % (k, len(best_dict[k]))) + sys.stdout.write("Unpaired t-tests-----------------------------------------------------\n") for idx, k in enumerate(keys): if len(keys) > 1: for jdx, j in enumerate(keys[idx+1:]): - #if len(bestDict[k]) == len(bestDict[j]): - T_t, p_t = _unpaired_ttest(bestDict[k], bestDict[j], equal_var=True) - T_f, p_f = _unpaired_ttest(bestDict[k], bestDict[j], equal_var=False) - T_rt, p_rt = _unpaired_ttest(numpy.round(bestDict[k], 3), numpy.round(bestDict[j], 3), equal_var=True) - T_rf, p_rf = _unpaired_ttest(numpy.round(bestDict[k], 3), numpy.round(bestDict[j], 3), equal_var=False) + t_true, p_true = stats.ttest_ind(best_dict[k], best_dict[j]) + rounded_t_true, rounded_p_true = stats.ttest_ind(numpy.round(best_dict[k], 3), + numpy.round(best_dict[j], 3)) + sys.stdout.write("Standard independent 2 sample test, equal population variance\n") - sys.stdout.write("%10s vs %10s" % (k,j)) + sys.stdout.write("%10s vs %10s" % (k, j)) sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%) \n" % - (T_t, p_t, p_t*100)) - sys.stdout.write("Rounded: ") - sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" % - (T_rt, p_rt, p_rt*100)) - sys.stdout.write("Welch's t-test, no equal population variance\n") - sys.stdout.write("%10s vs %10s" % (k,j)) - sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" % - (T_f, p_f, p_f*100)) + (t_true, p_true, p_true*100)) sys.stdout.write("Rounded: ") sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" % - (T_rf, p_rf, p_rf*100)) + (rounded_t_true, rounded_p_true, rounded_p_true*100)) + if StrictVersion(scipy.__version__) >= StrictVersion('0.11.0'): + print scipy.__version__ >= '0.11.0' + t_false, p_false = stats.ttest_ind(best_dict[k], best_dict[j], equal_var=False) + rounded_t_false, rounded_p_false = stats.ttest_ind(numpy.round(best_dict[k], 3), + numpy.round(best_dict[j], 3), + equal_var=False) + sys.stdout.write("Welch's t-test, no equal population variance\n") + sys.stdout.write("%10s vs %10s" % (k, j)) + sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" % + (t_false, p_false, p_false*100)) + sys.stdout.write("Rounded: ") + sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" % + (rounded_t_false, rounded_p_false, rounded_p_false*100)) + sys.stdout.write("Best Value-----------------------------------------------------------\n") for k in keys: - stdBest = numpy.std(bestDict[k]) - minBest = numpy.min(bestDict[k]) - maxBest = numpy.max(bestDict[k]) - meanBest = numpy.mean(bestDict[k]) + std_best = numpy.std(best_dict[k]) + min_best = numpy.min(best_dict[k]) + max_best = numpy.max(best_dict[k]) + mean_best = numpy.mean(best_dict[k]) sys.stdout.write("%10s: %10.5f (min: %10.5f, max: %10.5f, std: %5.3f)\n" % - (k, meanBest, minBest, maxBest, stdBest)) + (k, float(mean_best), float(min_best), max_best, float(std_best))) sys.stdout.write("Needed Trials--------------------------------------------------------\n") for k in keys: - stdIdx = numpy.std(idxDict[k]) - meanIdx = numpy.mean(idxDict[k]) - minIdx = numpy.min(idxDict[k]) - maxIdx = numpy.max(idxDict[k]) + std_idx = numpy.std(idx_dict[k]) + mean_idx = numpy.mean(idx_dict[k]) + min_idx = numpy.min(idx_dict[k]) + max_idx = numpy.max(idx_dict[k]) sys.stdout.write("%10s: %10.5f (min: %10.5f, max: %10.5f, std: %5.3f)\n" % - (k, meanIdx, minIdx, maxIdx, stdIdx)) + (k, float(mean_idx), float(min_idx), max_idx, float(std_idx))) sys.stdout.write("------------------------------------------------------------------------\n") From b5ef849c96b3ab0e22db8a813e0fff19375366ed Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Mon, 3 Mar 2014 13:28:51 +0100 Subject: [PATCH 047/139] REFACTOR: #20 Replace config section DEFAULT by HPOlib --- HPOlib/benchmarks/branin/config.cfg | 2 +- HPOlib/benchmarks/har6/config.cfg | 2 +- HPOlib/check_before_start.py | 2 +- HPOlib/config_parser/generalDefault.cfg | 10 +- HPOlib/config_parser/parse.py | 28 +- HPOlib/cv.py | 8 +- HPOlib/optimizers/smac/smac_2_06_01-dev.py | 4 +- .../smac/smac_2_06_01-dev_parser.py | 10 +- .../spearmint/spearmint_april2013_mod.py | 4 +- .../spearmint_april2013_mod_parser.py | 2 +- .../optimizers/tpe/hyperopt_august2013_mod.py | 4 +- .../tpe/hyperopt_august2013_mod_parser.py | 2 +- .../tpe/random_hyperopt_august2013_mod.py | 6 +- HPOlib/runsolver_wrapper.py | 32 +- HPOlib/wrapping.py | 29 +- HPOlib/wrapping_util.py | 2 +- tests/dummy_config.cfg | 2 +- tests/test_get_trial_index.pkl | 421 ------------------ tests/test_wrapping.py | 74 +-- 19 files changed, 87 insertions(+), 557 deletions(-) delete mode 100644 tests/test_get_trial_index.pkl diff --git a/HPOlib/benchmarks/branin/config.cfg b/HPOlib/benchmarks/branin/config.cfg index bf069411..1aabf8e4 100644 --- a/HPOlib/benchmarks/branin/config.cfg +++ b/HPOlib/benchmarks/branin/config.cfg @@ -1,7 +1,7 @@ [SMAC] p = smac_2_06_01-dev/params.txt -[DEFAULT] +[HPOLIB] function = branin.py numberOfJobs = 200 result_on_terminate = 1000 diff --git a/HPOlib/benchmarks/har6/config.cfg b/HPOlib/benchmarks/har6/config.cfg index 8354d2d1..e9e90e72 100644 --- a/HPOlib/benchmarks/har6/config.cfg +++ b/HPOlib/benchmarks/har6/config.cfg @@ -1,7 +1,7 @@ [SMAC] p = smac/params.txt -[DEFAULT] +[HPOLIB] function = har6.py numberOfJobs = 200 result_on_terminate = 1 diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index 95c64d9c..0b4fd672 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -93,7 +93,7 @@ def _check_function(experiment_dir): config_file = os.path.join(experiment_dir, "config.cfg") config = parse_config(config_file, allow_no_value=True) - path = config.get("DEFAULT", "function") + path = config.get("HPOLIB", "function") if not os.path.isabs(path): path = os.path.join(experiment_dir, path) diff --git a/HPOlib/config_parser/generalDefault.cfg b/HPOlib/config_parser/generalDefault.cfg index 271f34dc..a5ecc885 100644 --- a/HPOlib/config_parser/generalDefault.cfg +++ b/HPOlib/config_parser/generalDefault.cfg @@ -1,4 +1,4 @@ -[DEFAULT] +[HPOLIB] #Will be used for wrapping.py and SMAC algorithm = cv.py run_instance = runsolver_wrapper.py @@ -18,11 +18,3 @@ cpu_limit = 14400 # E.g. Information for Theano and cuda leading_runsolver_info = leading_algo_info = - -training_data = -training_targets = -test_data = -test_targets = -#only gfile, numpy and pickle supported -training_data_format = numpy -use_percentage = 100 diff --git a/HPOlib/config_parser/parse.py b/HPOlib/config_parser/parse.py index df49a09d..157d4011 100644 --- a/HPOlib/config_parser/parse.py +++ b/HPOlib/config_parser/parse.py @@ -47,27 +47,25 @@ def parse_config(config_fn, allow_no_value=True, optimizer_version=""): config_default.read(config_fn_default) # -------------------------------------------------------------------------- - # DEFAULT + # Defaults for HPOLIB # -------------------------------------------------------------------------- - # Set defaults for DEFAULT + # Set defaults for section HPOLIB for option in ('algorithm', 'run_instance', 'numberCV', 'leading_algo_info', 'numberOfConcurrentJobs', 'runsolver_time_limit', 'total_time_limit', 'memory_limit', - 'cpu_limit', 'leading_runsolver_info', 'training_data', - 'training_targets', 'test_data', 'test_targets', - 'training_data_format', 'use_percentage', 'max_crash_per_cv'): - if not config.has_option('DEFAULT', option): - config.set('DEFAULT', option, - config_default.get('DEFAULT', option)) - - if not config.has_option('DEFAULT', 'numberOfJobs') or \ - config.get('DEFAULT', 'numberOfJobs') == '': + 'cpu_limit', 'leading_runsolver_info', 'max_crash_per_cv'): + if not config.has_option('HPOLIB', option): + config.set('HPOLIB', option, + config_default.get('HPOLIB', option)) + + if not config.has_option('HPOLIB', 'numberOfJobs') or \ + config.get('HPOLIB', 'numberOfJobs') == '': raise Exception('numberOfJobs not specified in .cfg') - if not config.has_option('DEFAULT', 'result_on_terminate') or \ - config.get('DEFAULT', 'result_on_terminate') == '': + if not config.has_option('HPOLIB', 'result_on_terminate') or \ + config.get('HPOLIB', 'result_on_terminate') == '': raise Exception('No result_on_terminate specified in .cfg') - if not config.has_option('DEFAULT', 'function') or \ - config.get('DEFAULT', 'function') == '': + if not config.has_option('HPOLIB', 'function') or \ + config.get('HPOLIB', 'function') == '': raise Exception('No function specified in .cfg') # Load optimizer parsing module diff --git a/HPOlib/cv.py b/HPOlib/cv.py index 274145c7..8ffd7dce 100644 --- a/HPOlib/cv.py +++ b/HPOlib/cv.py @@ -108,9 +108,9 @@ def doCV(params, folds=10): # If a specified number of runs crashed, quit the whole cross validation # in order to save time. - worst_possible = cfg.getfloat("DEFAULT", "result_on_terminate") + worst_possible = cfg.getfloat("HPOLIB", "result_on_terminate") crashed_runs = np.nansum([0 if res != worst_possible else 1 for res in results]) - if crashed_runs >= cfg.getint("DEFAULT", "max_crash_per_cv"): + if crashed_runs >= cfg.getint("HPOLIB", "max_crash_per_cv"): logger.warning("Aborting CV because the number of crashes " \ "excceds the configured max_crash_per_cv value") return worst_possible @@ -129,7 +129,7 @@ def doCV(params, folds=10): # Do not return any kind of nan because this would break spearmint if not np.isfinite(mean): - mean = float(cfg.get("DEFAULT", "result_on_terminate")) + mean = float(cfg.get("HPOLIB", "result_on_terminate")) logger.info("Finished CV") return mean @@ -205,7 +205,7 @@ def main(job_id, params): cfg = load_experiment_config_file() # Load number of folds - folds = cfg.getint('DEFAULT', 'numberCV') + folds = cfg.getint('HPOLIB', 'numberCV') params = flatten_parameter_dict(params) diff --git a/HPOlib/optimizers/smac/smac_2_06_01-dev.py b/HPOlib/optimizers/smac/smac_2_06_01-dev.py index 57adec67..d2b66a83 100644 --- a/HPOlib/optimizers/smac/smac_2_06_01-dev.py +++ b/HPOlib/optimizers/smac/smac_2_06_01-dev.py @@ -210,12 +210,12 @@ def main(config, options, experiment_dir, **kwargs): # Copy the smac search space and create the instance information fh = open(os.path.join(optimizer_dir, 'train.txt'), "w") - for i in range(config.getint('DEFAULT', 'numberCV')): + for i in range(config.getint('HPOLIB', 'numberCV')): fh.write(str(i) + "\n") fh.close() fh = open(os.path.join(optimizer_dir, 'test.txt'), "w") - for i in range(config.getint('DEFAULT', 'numberCV')): + for i in range(config.getint('HPOLIB', 'numberCV')): fh.write(str(i) + "\n") fh.close() diff --git a/HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py b/HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py index ac3f4e2b..040363f4 100644 --- a/HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py +++ b/HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py @@ -59,17 +59,17 @@ def add_default(config): # special cases if not config.has_option('SMAC', 'cutoffTime'): config.set('SMAC', 'cutoffTime', - str(config.getint('DEFAULT', 'runsolver_time_limit') + 100)) + str(config.getint('HPOLIB', 'runsolver_time_limit') + 100)) if not config.has_option('SMAC', 'algoExec'): config.set('SMAC', 'algoExec', - config.get('DEFAULT', 'run_instance')) + config.get('HPOLIB', 'run_instance')) if not config.has_option('SMAC', 'totalNumRunsLimit'): config.set('SMAC', 'totalNumRunsLimit', - str(config.getint('DEFAULT', 'numberOfJobs') * - config.getint('DEFAULT', 'numberCV'))) + str(config.getint('HPOLIB', 'numberOfJobs') * + config.getint('HPOLIB', 'numberCV'))) if not config.has_option('SMAC', 'numConcurrentAlgoExecs'): config.set('SMAC', 'numConcurrentAlgoExecs', - config.get('DEFAULT', 'numberOfConcurrentJobs')) + config.get('HPOLIB', 'numberOfConcurrentJobs')) path_to_optimizer = smac_config.get('SMAC', 'path_to_optimizer') if not os.path.isabs(path_to_optimizer): diff --git a/HPOlib/optimizers/spearmint/spearmint_april2013_mod.py b/HPOlib/optimizers/spearmint/spearmint_april2013_mod.py index 59529f0e..5bf10154 100644 --- a/HPOlib/optimizers/spearmint/spearmint_april2013_mod.py +++ b/HPOlib/optimizers/spearmint/spearmint_april2013_mod.py @@ -43,7 +43,7 @@ def build_spearmint_call(config, options, optimizer_dir): call = 'python ' + os.path.join(config.get('SPEARMINT', 'path_to_optimizer'), 'spearmint_sync.py') call = ' '.join([call, optimizer_dir, '--config', config.get('SPEARMINT', 'config'), - '--max-concurrent', config.get('DEFAULT', 'numberOfConcurrentJobs'), + '--max-concurrent', config.get('HPOLIB', 'numberOfConcurrentJobs'), '--max-finished-jobs', config.get('SPEARMINT', 'max_finished_jobs'), '--polling-time', config.get('SPEARMINT', 'spearmint_polling_time'), '--grid-size', config.get('SPEARMINT', 'grid_size'), @@ -76,7 +76,7 @@ def restore(config, optimizer_dir, **kwargs): exp_grid = cPickle.load(fh) fh.close() complete_runs = np.sum(exp_grid['status'] == 3) - restored_runs = complete_runs * config.getint('DEFAULT', 'numberCV') + restored_runs = complete_runs * config.getint('HPOLIB', 'numberCV') try: os.remove(os.path.join(optimizer_dir, "expt-grid.pkl.lock")) except OSError: diff --git a/HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py b/HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py index e07fe390..4d3c27e2 100644 --- a/HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py +++ b/HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py @@ -62,7 +62,7 @@ def add_default(config): # GENERAL if not config.has_option('SPEARMINT', 'max_finished_jobs'): config.set('SPEARMINT', 'max_finished_jobs', - config.get('DEFAULT', 'numberOfJobs')) + config.get('HPOLIB', 'numberOfJobs')) path_to_optimizer = spearmint_config.get('SPEARMINT', 'path_to_optimizer') if not os.path.isabs(path_to_optimizer): diff --git a/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py b/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py index 0cb3e7f8..8d493e25 100644 --- a/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py +++ b/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py @@ -36,7 +36,7 @@ def build_tpe_call(config, options, optimizer_dir): "python " + os.path.dirname(os.path.realpath(__file__)) + \ "/tpecall.py" call = ' '.join([call, '-p', config.get('TPE', 'space'), - "-a", config.get('DEFAULT', 'algorithm'), + "-a", config.get('HPOLIB', 'algorithm'), "-m", config.get('TPE', 'numberEvals'), "-s", str(options.seed)]) if options.restore: @@ -66,7 +66,7 @@ def restore(config, optimizer_dir, **kwargs): # Assumes that all not valid states states are marked crashed if trial['state'] == 2: complete_runs += 1 - restored_runs = complete_runs * config.getint('DEFAULT', 'numberCV') + restored_runs = complete_runs * config.getint('HPOLIB', 'numberCV') return restored_runs diff --git a/HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py b/HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py index 89cdea8d..ea9f0e29 100644 --- a/HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py +++ b/HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py @@ -52,7 +52,7 @@ def add_default(config): if not config.has_option('TPE', 'numberEvals'): config.set('TPE', 'numberEvals', - config.get('DEFAULT', 'numberOfJobs')) + config.get('HPOLIB', 'numberOfJobs')) path_to_optimizer = tpe_config.get('TPE', 'path_to_optimizer') if not os.path.isabs(path_to_optimizer): diff --git a/HPOlib/optimizers/tpe/random_hyperopt_august2013_mod.py b/HPOlib/optimizers/tpe/random_hyperopt_august2013_mod.py index 856c4528..4344fcde 100644 --- a/HPOlib/optimizers/tpe/random_hyperopt_august2013_mod.py +++ b/HPOlib/optimizers/tpe/random_hyperopt_august2013_mod.py @@ -42,8 +42,8 @@ def build_random_call(config, options, optimizer_dir): 'python ' + os.path.dirname(os.path.realpath(__file__)) + \ '/tpecall.py' call = ' '.join([call, '-p', config.get('TPE', 'space'), - '-a', config.get('DEFAULT', 'algorithm'), - '-m', config.get('DEFAULT', 'numberOfJobs'), + '-a', config.get('HPOLIB', 'algorithm'), + '-m', config.get('HPOLIB', 'numberOfJobs'), '-s', str(options.seed), '--random']) if options.restore: call = ' '.join([call, '-r']) @@ -66,7 +66,7 @@ def restore(config, optimizer_dir, **kwargs): # Assumes that all states no valid state is marked crashed if trial['state'] == 2: complete_runs += 1 - restored_runs = complete_runs * config.getint('DEFAULT', 'numberCV') + restored_runs = complete_runs * config.getint('HPOLIB', 'numberCV') return restored_runs diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 09c0c016..7864ca0f 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -191,11 +191,11 @@ def parse_command_line(): def get_function_filename(cfg): - if os.path.isabs(cfg.get("DEFAULT", "function")): - fn_path = cfg.get("DEFAULT", "function") + if os.path.isabs(cfg.get("HPOLIB", "function")): + fn_path = cfg.get("HPOLIB", "function") else: - fn_path = cfg.get("DEFAULT", "function") - fn_path_parent = os.path.join("..", cfg.get("DEFAULT", "function")) + fn_path = cfg.get("HPOLIB", "function") + fn_path_parent = os.path.join("..", cfg.get("HPOLIB", "function")) fn_name, ext = os.path.splitext(os.path.basename(fn_path)) try: fn = imp.load_source(fn_name, fn_path) @@ -221,18 +221,18 @@ def get_function_filename(cfg): def make_command(cfg, fold, param_string, run_instance_output): - time_limit = cfg.getint('DEFAULT', 'runsolver_time_limit') - memory_limit = cfg.getint('DEFAULT', 'memory_limit') - cpu_limit = cfg.getint('DEFAULT', 'cpu_limit') + time_limit = cfg.getint('HPOLIB', 'runsolver_time_limit') + memory_limit = cfg.getint('HPOLIB', 'memory_limit') + cpu_limit = cfg.getint('HPOLIB', 'cpu_limit') fn_filename = get_function_filename(cfg) - python_cmd = cfg.get("DEFAULT", "leading_algo_info") + " python " + \ + python_cmd = cfg.get("HPOLIB", "leading_algo_info") + " python " + \ fn_filename + " --fold %d --folds %d --params %s" % \ - (fold, cfg.getint("DEFAULT", "numberCV"), param_string) + (fold, cfg.getint("HPOLIB", "numberCV"), param_string) # Do not write the actual task in quotes because runsolver will not work # then; also we need use-pty antd timestamp so that the "solver" output # is flushed to the output directory delay = 0 - cmd = cfg.get("DEFAULT", "leading_runsolver_info") + \ + cmd = cfg.get("HPOLIB", "leading_runsolver_info") + \ " runsolver -o %s --timestamp --use-pty -W %d -C %d -M %d -d %d %s" \ % (run_instance_output, time_limit, cpu_limit, memory_limit, delay, python_cmd) @@ -254,21 +254,21 @@ def parse_output_files(cfg, run_instance_output, runsolver_output_file): if error is None and result_string is None: additional_data = "No result string returned. Please have a look " \ "at " + run_instance_output - rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("DEFAULT", + rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("HPOLIB", "result_on_terminate"), additional_data) os.remove(runsolver_output_file) elif error is None and result_array[3] != "SAT": additional_data = "Please have a look at " + run_instance_output + "."\ "The output status is not \"SAT\"" - rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("DEFAULT", + rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("HPOLIB", "result_on_terminate"), additional_data) os.remove(runsolver_output_file) elif error is None and not np.isfinite(float(result_array[6].strip(","))): additional_data = "Response value is not finite. Please have a look " \ "at " + run_instance_output - rval = (cpu_time, wallclock_time, "UNSAT", cfg.getfloat("DEFAULT", + rval = (cpu_time, wallclock_time, "UNSAT", cfg.getfloat("HPOLIB", "result_on_terminate"), additional_data) elif error is None: @@ -276,10 +276,10 @@ def parse_output_files(cfg, run_instance_output, runsolver_output_file): os.remove(run_instance_output) os.remove(runsolver_output_file) rval = (cpu_time, wallclock_time, "SAT", float(result_array[6].strip(",")), - cfg.get("DEFAULT", "function")) + cfg.get("HPOLIB", "function")) else: - rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("DEFAULT", + rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("HPOLIB", "result_on_terminate"), error + " Please have a look at " + runsolver_output_file) # It is useful to have the run_instance_output for debugging @@ -342,7 +342,7 @@ def main(): experiment.set_one_fold_complete(trial_index, fold, result, wallclock_time) elif status == "CRASHED" or status == "UNSAT": - result = cfg.getfloat("DEFAULT", "result_on_terminate") + result = cfg.getfloat("HPOLIB", "result_on_terminate") experiment.set_one_fold_crashed(trial_index, fold, result, wallclock_time) else: diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 0e2cc349..39eeb364 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -107,6 +107,9 @@ def use_arg_parser(): parser.add_argument("-t", "--title", action="store", type=str, dest="title", default=None, help="A title for the experiment") + parser.add_argument("--cwd", action="store", type=str, dest="working_dir", + default=None, help="Change the working directory to " + " prior to running the experiment") parser.add_argument("-r", "--restore", action="store", type=str, dest="restore", default=None, help="Restore the state from a given directory") @@ -172,19 +175,18 @@ def override_config_with_cli_arguments(config, config_overrides): return config -def save_config_to_file(fn_handle, config): - for section in config.sections(): - fn_handle.write("[DEFAULT]\n") +def save_config_to_file(file_handle, config): + if len(config.defaults()) > 0: + file_handle.write("[DEFAULT]\n") for key in config.defaults(): - if config.get(section, key) is None or \ - config.get(section, key) != "": - fn_handle.write(key + " = " + config.get("DEFAULT", key) + "\n") + file_handle.write(key + " = " + config.get("DEFAULT", key) + "\n") - fn_handle.write("[" + section + "]\n") + for section in config.sections(): + file_handle.write("[" + section + "]\n") for key in config.options(section): if config.get(section, key) is None or \ config.get(section, key) != "": - fn_handle.write(key + " = " + config.get(section, key) + "\n") + file_handle.write(key + " = " + config.get(section, key) + "\n") def main(): @@ -192,6 +194,9 @@ def main(): comments inside this function and the general HPOlib documentation.""" args, unknown = use_arg_parser() + if args.working_dir: + os.chdir(args.working_dir) + experiment_dir = os.getcwd() check_before_start.check_zeroth(experiment_dir) @@ -243,7 +248,7 @@ def main(): cmd += optimizer_call with open(os.path.join(optimizer_dir_in_experiment, "config.cfg"), "w") as f: - config.set("DEFAULT", "is_not_original_config_file", "True") + config.set("HPOLIB", "is_not_original_config_file", "True") save_config_to_file(f, config) # _check_function @@ -255,9 +260,9 @@ def main(): os.remove(os.path.join(optimizer_dir_in_experiment, optimizer + ".pkl.lock")) except OSError: pass - folds = config.getint('DEFAULT', 'numberCV') + folds = config.getint('HPOLIB', 'numberCV') trials = Experiment.Experiment(optimizer_dir_in_experiment, optimizer, folds=folds, - max_wallclock_time=config.get('DEFAULT', + max_wallclock_time=config.get('HPOLIB', 'cpu_limit'), title=args.title) trials.optimizer = optimizer_version @@ -280,7 +285,7 @@ def main(): pass else: trials.get_trial_from_id(_id)['instance_results'][fold] = \ - config.getfloat('DEFAULT', 'result_on_terminate') + config.getfloat('HPOLIB', 'result_on_terminate') # Pretty sure we need this here: trials.get_trial_from_id(_id)['instance_status'][fold] = \ Experiment.BROKEN_STATE diff --git a/HPOlib/wrapping_util.py b/HPOlib/wrapping_util.py index 5da8e6d5..ef676cd3 100644 --- a/HPOlib/wrapping_util.py +++ b/HPOlib/wrapping_util.py @@ -85,7 +85,7 @@ def load_experiment_config_file(): try: cfg_filename = "config.cfg" cfg = parse_config(cfg_filename, allow_no_value=True) - if not cfg.has_option("DEFAULT", "is_not_original_config_file"): + if not cfg.has_option("HPOLIB", "is_not_original_config_file"): logger.critical("Config file in directory %s seems to be an" " original config which was not created by wrapping.py. " "Please contact the HPOlib maintainer to solve this issue.") diff --git a/tests/dummy_config.cfg b/tests/dummy_config.cfg index a2bf4027..8a0ba2db 100644 --- a/tests/dummy_config.cfg +++ b/tests/dummy_config.cfg @@ -1,4 +1,4 @@ -[DEFAULT] +[HPOLIB] numberOfJobs = 1 result_on_terminate = 1 function = 1 \ No newline at end of file diff --git a/tests/test_get_trial_index.pkl b/tests/test_get_trial_index.pkl deleted file mode 100644 index 98aaf140..00000000 --- a/tests/test_get_trial_index.pkl +++ /dev/null @@ -1,421 +0,0 @@ -(dp1 -S'cv_endtime' -p2 -(lp3 -sS'folds' -p4 -I1 -sS'cv_starttime' -p5 -(lp6 -sS'endtime' -p7 -(lp8 -sS'optimizer' -p9 -NsS'optimizer_time' -p10 -(lp11 -sS'instance_order' -p12 -(lp13 -(I0 -I0 -tp14 -a(I1 -I0 -tp15 -a(I2 -I0 -tp16 -a(I3 -I0 -tp17 -a(I4 -I0 -tp18 -asS'max_wallclock_time' -p19 -F1.7976931348623157e+308 -sS'title' -p20 -NsS'trials' -p21 -(lp22 -(dp23 -S'status' -p24 -I3 -sS'std' -p25 -cnumpy.core.multiarray -scalar -p26 -(cnumpy -dtype -p27 -(S'f8' -I0 -I1 -tRp28 -(I3 -S'<' -NNNI-1 -I-1 -I0 -tbS'\x00\x00\x00\x00\x00\x00\x00\x00' -tRp29 -sS'instance_durations' -p30 -cnumpy.core.multiarray -_reconstruct -p31 -(cnumpy -ndarray -p32 -(I0 -tS'b' -tRp33 -(I1 -(I1 -tg28 -I00 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tbsS'instance_results' -p34 -g31 -(g32 -(I0 -tS'b' -tRp35 -(I1 -(I1 -tg28 -I00 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tbsS'test_error' -p36 -Fnan -sS'params' -p37 -(dp38 -S'x' -S'1' -ssS'result' -p39 -g26 -(g28 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tRp40 -sS'duration' -p41 -g26 -(g28 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tRp42 -sS'instance_status' -p43 -g31 -(g32 -(I0 -tS'b' -tRp44 -(I1 -(I1 -tg27 -(S'i4' -I0 -I1 -tRp45 -(I3 -S'<' -NNNI-1 -I-1 -I0 -tbI00 -S'\x03\x00\x00\x00' -tbsa(dp46 -g24 -I3 -sg25 -g26 -(g28 -S'\x00\x00\x00\x00\x00\x00\x00\x00' -tRp47 -sg30 -g31 -(g32 -(I0 -tS'b' -tRp48 -(I1 -(I1 -tg28 -I00 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tbsg34 -g31 -(g32 -(I0 -tS'b' -tRp49 -(I1 -(I1 -tg28 -I00 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tbsg36 -Fnan -sg37 -(dp50 -S'x' -S'2' -ssg39 -g26 -(g28 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tRp51 -sg41 -g26 -(g28 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tRp52 -sg43 -g31 -(g32 -(I0 -tS'b' -tRp53 -(I1 -(I1 -tg45 -I00 -S'\x03\x00\x00\x00' -tbsa(dp54 -g24 -I3 -sg25 -g26 -(g28 -S'\x00\x00\x00\x00\x00\x00\x00\x00' -tRp55 -sg30 -g31 -(g32 -(I0 -tS'b' -tRp56 -(I1 -(I1 -tg28 -I00 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tbsg34 -g31 -(g32 -(I0 -tS'b' -tRp57 -(I1 -(I1 -tg28 -I00 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tbsg36 -Fnan -sg37 -(dp58 -S'x' -S'3' -ssg39 -g26 -(g28 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tRp59 -sg41 -g26 -(g28 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tRp60 -sg43 -g31 -(g32 -(I0 -tS'b' -tRp61 -(I1 -(I1 -tg45 -I00 -S'\x03\x00\x00\x00' -tbsa(dp62 -g24 -I3 -sg25 -g26 -(g28 -S'\x00\x00\x00\x00\x00\x00\x00\x00' -tRp63 -sg30 -g31 -(g32 -(I0 -tS'b' -tRp64 -(I1 -(I1 -tg28 -I00 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tbsg34 -g31 -(g32 -(I0 -tS'b' -tRp65 -(I1 -(I1 -tg28 -I00 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tbsg36 -Fnan -sg37 -(dp66 -S'x' -S'4' -ssg39 -g26 -(g28 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tRp67 -sg41 -g26 -(g28 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tRp68 -sg43 -g31 -(g32 -(I0 -tS'b' -tRp69 -(I1 -(I1 -tg45 -I00 -S'\x03\x00\x00\x00' -tbsa(dp70 -g24 -I3 -sg25 -g26 -(g28 -S'\x00\x00\x00\x00\x00\x00\x00\x00' -tRp71 -sg30 -g31 -(g32 -(I0 -tS'b' -tRp72 -(I1 -(I1 -tg28 -I00 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tbsg34 -g31 -(g32 -(I0 -tS'b' -tRp73 -(I1 -(I1 -tg28 -I00 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tbsg36 -Fnan -sg37 -(dp74 -S'x' -S'5' -ssg39 -g26 -(g28 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tRp75 -sg41 -g26 -(g28 -S'\x00\x00\x00\x00\x00\x00\xf0?' -tRp76 -sg43 -g31 -(g32 -(I0 -tS'b' -tRp77 -(I1 -(I1 -tg45 -I00 -S'\x03\x00\x00\x00' -tbsa(dp78 -g24 -I0 -sg25 -Fnan -sg30 -g31 -(g32 -(I0 -tS'b' -tRp79 -(I1 -(I1 -tg28 -I00 -S'\x00\x00\x00\x00\x00\x00\xf8\x7f' -tbsg34 -g31 -(g32 -(I0 -tS'b' -tRp80 -(I1 -(I1 -tg28 -I00 -S'\x00\x00\x00\x00\x00\x00\xf8\x7f' -tbsg36 -Fnan -sg37 -g58 -sg39 -Fnan -sg41 -Fnan -sg43 -g31 -(g32 -(I0 -tS'b' -tRp81 -(I1 -(I1 -tg45 -I00 -S'\x00\x00\x00\x00' -tbsasS'total_wallclock_time' -p82 -I5 -sS'experiment_name' -p83 -S'test_get_trial_index' -p84 -sS'starttime' -p85 -(lp86 -s. \ No newline at end of file diff --git a/tests/test_wrapping.py b/tests/test_wrapping.py index 93528a43..db64419f 100644 --- a/tests/test_wrapping.py +++ b/tests/test_wrapping.py @@ -80,11 +80,11 @@ def test_build_smac_call(self): def test_use_option_parser_no_optimizer(self): # Test not specifying an optimizer but random other options sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet'] - self.assertRaises(SystemExit, wrapping.use_option_parser) + self.assertRaises(SystemExit, wrapping.use_arg_parser) def test_use_option_parser_the_right_way(self): sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet', '-o', 'SMAC'] - args, unknown = wrapping.use_option_parser() + args, unknown = wrapping.use_arg_parser() self.assertEqual(args.optimizer, 'SMAC') self.assertEqual(args.seed, 1) self.assertEqual(args.title, 'DBNet') @@ -92,40 +92,34 @@ def test_use_option_parser_the_right_way(self): def test_use_option_parser_with_config(self): sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet', '-o', 'SMAC', - '--DEFAULT:total_time_limit', '3600'] - args, unknown = wrapping.use_option_parser() + '--HPOLIB:total_time_limit', '3600'] + args, unknown = wrapping.use_arg_parser() self.assertEqual(len(unknown), 2) - config = parse.parse_config("dummy_config.cfg", - allow_no_value=True, - optimizer_module="smac") + config = parse.parse_config("dummy_config.cfg", allow_no_value=True) config_args = wrapping.parse_config_values_from_unknown_arguments( unknown, config) - self.assertEqual(vars(config_args)['DEFAULT:total_time_limit'], + self.assertEqual(vars(config_args)['HPOLIB:total_time_limit'], '3600') def test_override_config_with_cli_arguments(self): - unknown = ['--DEFAULT:total_time_limit', '50', - '--DEFAULT:numberofjobs', '2'] - config = parse.parse_config("dummy_config.cfg", - allow_no_value=True, - optimizer_module="smac") + unknown = ['--HPOLIB:total_time_limit', '50', + '--HPOLIB:numberofjobs', '2'] + config = parse.parse_config("dummy_config.cfg", allow_no_value=True) config_args = wrapping.parse_config_values_from_unknown_arguments( unknown, config) new_config = wrapping.override_config_with_cli_arguments(config, config_args) - self.assertEqual(new_config.get('DEFAULT', 'numberofjobs'), '2') + self.assertEqual(new_config.get('HPOLIB', 'numberofjobs'), '2') def test_save_config_to_file(self): - unknown = ['--DEFAULT:total_time_limit', '50', - '--DEFAULT:numberofjobs', '2'] - config = parse.parse_config("dummy_config.cfg", - allow_no_value=True, - optimizer_module="smac") + unknown = ['--HPOLIB:total_time_limit', '50', + '--HPOLIB:numberofjobs', '2'] + config = parse.parse_config("dummy_config.cfg", allow_no_value=True) string_stream = StringIO.StringIO() wrapping.save_config_to_file(string_stream, config) file_content = string_stream.getvalue() - asserted_file_content = "[DEFAULT]\n" \ + asserted_file_content = "[HPOLIB]\n" \ "numberofjobs = 1\n" \ "result_on_terminate = 1\n"\ "function = 1\n"\ @@ -137,45 +131,7 @@ def test_save_config_to_file(self): "total_time_limit = 3600\n"\ "memory_limit = 2000\n"\ "cpu_limit = 14400\n"\ - "training_data_format = numpy\n"\ - "use_percentage = 100\n"\ - "max_crash_per_cv = 3\n"\ - "optimizer_version = smac_2_06_01-dev\n"\ - "[SMAC]\n"\ - "numrun = 0\n"\ - "instancefile = train.txt\n"\ - "intrainstanceobj = MEAN\n"\ - "runobj = QUALITY\n"\ - "testinstancefile = test.txt\n"\ - "p = smac/params.pcs\n"\ - "rf_full_tree_bootstrap = False\n"\ - "rf_split_min = 10\n"\ - "adaptivecapping = false\n"\ - "maxincumbentruns = 2000\n"\ - "numiterations = 2147483647\n"\ - "runtimelimit = 2147483647\n"\ - "deterministic = True\n"\ - "retrytargetalgorithmruncount = 0\n"\ - "intensification_percentage = 0\n"\ - "cutofftime = 3700\n"\ - "algoexec = runsolver_wrapper.py\n"\ - "totalnumrunslimit = 1\n"\ - "numconcurrentalgoexecs = 1\n"\ - "numberofjobs = 1\n"\ - "result_on_terminate = 1\n"\ - "function = 1\n"\ - "algorithm = cv.py\n"\ - "run_instance = runsolver_wrapper.py\n"\ - "numbercv = 1\n"\ - "numberofconcurrentjobs = 1\n"\ - "runsolver_time_limit = 3600\n"\ - "total_time_limit = 3600\n"\ - "memory_limit = 2000\n"\ - "cpu_limit = 14400\n"\ - "training_data_format = numpy\n"\ - "use_percentage = 100\n"\ - "max_crash_per_cv = 3\n"\ - "optimizer_version = smac_2_06_01-dev\n" + "max_crash_per_cv = 3\n" self.assertEqual(asserted_file_content, file_content) string_stream.close() From ef8e9c9eae04de270c9c7fa92ec67cf826364999 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 3 Mar 2014 13:53:57 +0100 Subject: [PATCH 048/139] REFACTOR: #16, #1: plotParam.py --- .../Plotting/{plot_param.py => plotParam.py} | 81 ++++++++++++------- 1 file changed, 52 insertions(+), 29 deletions(-) rename HPOlib/Plotting/{plot_param.py => plotParam.py} (63%) diff --git a/HPOlib/Plotting/plot_param.py b/HPOlib/Plotting/plotParam.py similarity index 63% rename from HPOlib/Plotting/plot_param.py rename to HPOlib/Plotting/plotParam.py index 5d1cfb5a..44b74971 100644 --- a/HPOlib/Plotting/plot_param.py +++ b/HPOlib/Plotting/plotParam.py @@ -1,7 +1,26 @@ #!/usr/bin/env python +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from argparse import ArgumentParser + import cPickle -import optparse import re import sys @@ -10,9 +29,9 @@ import numpy as np +from HPOlib.Plotting import plot_util __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] -__license__ = "3-clause BSD License" __contact__ = "automl.org" @@ -111,48 +130,52 @@ def plot_params(value_list, result_list, name, save="", title="", jitter=0): def main(): - usage = "python plot_param.py WhatIsThis * [options]" - parser = optparse.OptionParser(usage) - parser.add_option("-s", "--save", dest="save", default="", - help="Where to save plot instead of showing it?") - parser.add_option("-p", "--parameter", dest="param", default="", - help="Which parameter to plot") - parser.add_option("-j", "--jitter", dest="jitter", default=0, type=float, - help="Jitter data?") - parser.add_option("-t", "--title", dest="title", default="", - help="Choose a supertitle for the plot") - (opts, args) = parser.parse_args() - sys.stdout.write("\nFound " + str(len(args)) + " argument[s]...\n") - - title = opts.title + prog = "python plot_param.py WhatIsThis * " + description = "Plot one param wrt to minfunction value" + + parser = ArgumentParser(description=description, prog=prog) + parser.add_argument("-s", "--save", dest="save", default="", + help="Where to save plot instead of showing it?") + parser.add_argument("-p", "--parameter", dest="param", default="", + help="Which parameter to plot") + parser.add_argument("-j", "--jitter", dest="jitter", default=0, type=float, + help="Jitter data?") + parser.add_argument("-t", "--title", dest="title", default="", + help="Choose a supertitle for the plot") + + args, unknown = parser.parse_known_args() + + sys.stdout.write("\nFound " + str(len(unknown)) + " argument[s]...\n") + pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) + + if len(name_list) > 1: + raise NotImplementedError("No support for more than one experiment") + + param = "-" + args.param value_list = list() result_list = list() param_set = set() - for fn in args: - if not ".pkl" in fn: - print "This is not a .pkl file: %s" % (fn,) - continue - print "Loading ", fn - fh = open(fn, "r") + for pkl in pkl_list[0]: + fh = open(pkl, "r") trials = cPickle.load(fh) fh.close() - for t in trials["trials"]: - if opts.param in t["params"]: - k, value = translate_para(opts.param, t["params"][opts.param]) + if param in t["params"]: + k, value = translate_para(args.param, t["params"][param]) value_list.append(float(value.strip("'"))) result_list.append(t["result"]) param_set.update(t["params"].keys()) if len(value_list) == 0: - print("No values found for param '%s', Available params:\n%s" % (opts.param, "\n".join([p for p in param_set]))) + print("No values found for param '%s', Available params:\n%s" % + (args.param, "\n".join([p[1:] for p in param_set]))) sys.exit(1) else: - print "Found %s values for %s" % (str(len(value_list)), opts.param) + print "Found %s values for %s" % (str(len(value_list)), args.param) - plot_params(value_list=value_list, result_list=result_list, name=opts.param, save=opts.save, title=title, - jitter=opts.jitter) + plot_params(value_list=value_list, result_list=result_list, name=args.param, save=args.save, title=args.title, + jitter=args.jitter) if __name__ == "__main__": main() From 44314c9bfe542999ab49ef01bd361120a9519bf6 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 3 Mar 2014 14:18:47 +0100 Subject: [PATCH 049/139] REFACTOR #1: separate cli from plotting method --- HPOlib/Plotting/plotBoxWhisker.py | 43 ++++++------ HPOlib/Plotting/plotOptimizerOverhead.py | 71 ++++++++++--------- HPOlib/Plotting/plotParam.py | 54 +++++++-------- HPOlib/Plotting/plotTrace.py | 54 +++++++-------- HPOlib/Plotting/plotTraceWithStd_perEval.py | 75 ++++++++++---------- HPOlib/Plotting/plotTraceWithStd_perTime.py | 76 +++++++++++---------- HPOlib/Plotting/statistics.py | 24 +++---- 7 files changed, 199 insertions(+), 198 deletions(-) diff --git a/HPOlib/Plotting/plotBoxWhisker.py b/HPOlib/Plotting/plotBoxWhisker.py index c7ce905b..b5882187 100644 --- a/HPOlib/Plotting/plotBoxWhisker.py +++ b/HPOlib/Plotting/plotBoxWhisker.py @@ -104,7 +104,26 @@ def plot_box_whisker(best_trials, name_list, title="", save="", y_min=0, y_max=0 show() -def main(): +def main(pkl_list, name_list, title="", save="", y_min=0, y_max=0): + + best_trials = list() + for i in range(len(name_list)): + best_trials.append(list()) + for pkl in pkl_list[i]: + fh = open(pkl, "r") + trials = cPickle.load(fh) + fh.close() + best_trials[i].append(plot_util.get_best(trials)) + + plot_box_whisker(best_trials=best_trials, name_list=name_list, + title=title, save=save, y_min=y_min, y_max=y_max) + + if save != "": + sys.stdout.write("Saving plot to " + save + "\n") + else: + sys.stdout.write("..Done\n") + +if __name__ == "__main__": prog = "python plotBoxWhisker.py WhatIsThis [WhatIsThis ]" description = "Plot a Box whisker plot for many experiments. The box covers lower to upper quartile." @@ -123,23 +142,7 @@ def main(): sys.stdout.write("\nFound " + str(len(unknown)) + " arguments...") - pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) - best_trials = list() - for i in range(len(name_list)): - best_trials.append(list()) - for pkl in pkl_list[i]: - fh = open(pkl, "r") - trials = cPickle.load(fh) - fh.close() - best_trials[i].append(plot_util.get_best(trials)) - - plot_box_whisker(best_trials=best_trials, name_list=name_list, - title=args.title, save=args.save, y_min=args.min, y_max=args.max) + pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) - if args.save != "": - sys.stdout.write("Saving plot to " + args.save + "\n") - else: - sys.stdout.write("..Done\n") - -if __name__ == "__main__": - main() + main(pkl_list=pkl_list_main, name_list=name_list_main, title=args.title, save=args.save, + y_min=args.min, y_max=args.max) diff --git a/HPOlib/Plotting/plotOptimizerOverhead.py b/HPOlib/Plotting/plotOptimizerOverhead.py index a22df97d..d7f905d4 100644 --- a/HPOlib/Plotting/plotOptimizerOverhead.py +++ b/HPOlib/Plotting/plotOptimizerOverhead.py @@ -106,34 +106,7 @@ def plot_time_trace(time_dict, name_list, title="", log=True, save="", y_max=0, plt.show() -def main(): - prog = "python plotOptimizerOverhead.py WhatIsThis [WhatIsThis ]" - description = "Plot a Trace with std for multiple experiments" - - parser = ArgumentParser(description=description, prog=prog) - - # General Options - parser.add_argument("-l", "--nolog", action="store_true", dest="log", - default=False, help="Do NOT plot on log scale") - parser.add_argument("--max", type=float, dest="max", - default=0, help="Maximum of the plot") - parser.add_argument("--min", type=float, dest="min", - default=0, help="Minimum of the plot") - parser.add_argument("-s", "--save", dest="save", - default="", help="Where to save plot instead of showing it?") - parser.add_argument("-t", "--title", dest="title", - default="", help="Choose a supertitle for the plot") - - # Options which are available only for this plot - parser.add_argument("-a", "--autofill", action="store_true", dest="autofill", - default=False, help="Fill trace automatically") - - args, unknown = parser.parse_known_args() - - sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n") - - pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) - +def main(pkl_list, name_list, autofill, title="", log=False, save="", y_min=0, y_max=0): times_dict = dict() for exp in range(len(name_list)): times_dict[name_list[exp][0]] = list() @@ -175,22 +148,48 @@ def main(): for key in times_dict.keys(): max_len = max([len(ls) for ls in times_dict[key]]) for t in range(len(times_dict[key])): - if len(times_dict[key][t]) < max_len and args.autofill: + if len(times_dict[key][t]) < max_len and autofill: diff = max_len - len(times_dict[key][t]) # noinspection PyUnusedLocal times_dict[key][t] = np.append(times_dict[key][t], [times_dict[key][t][-1] for x in range(diff)]) - elif len(times_dict[key][t]) < max_len and not args.autofill: + elif len(times_dict[key][t]) < max_len and not autofill: raise ValueError("(%s != %s), Traces do not have the same length, please use -a" % (str(max_len), str(len(times_dict[key][t])))) - title = args.title - plot_time_trace(times_dict, name_list, title=title, log=not args.log, save=args.save, - y_min=args.min, y_max=args.max) + plot_time_trace(times_dict, name_list, title=title, log=not log, save=save, + y_min=y_min, y_max=y_max) - if args.save != "": - sys.stdout.write("Saved plot to " + args.save + "\n") + if save != "": + sys.stdout.write("Saved plot to " + save + "\n") else: sys.stdout.write("..Done\n") if __name__ == "__main__": - main() + prog = "python plotOptimizerOverhead.py WhatIsThis [WhatIsThis ]" + description = "Plot a Trace with std for multiple experiments" + + parser = ArgumentParser(description=description, prog=prog) + + # General Options + parser.add_argument("-l", "--nolog", action="store_true", dest="log", + default=False, help="Do NOT plot on log scale") + parser.add_argument("--max", type=float, dest="max", + default=0, help="Maximum of the plot") + parser.add_argument("--min", type=float, dest="min", + default=0, help="Minimum of the plot") + parser.add_argument("-s", "--save", dest="save", + default="", help="Where to save plot instead of showing it?") + parser.add_argument("-t", "--title", dest="title", + default="", help="Choose a supertitle for the plot") + + # Options which are available only for this plot + parser.add_argument("-a", "--autofill", action="store_true", dest="autofill", + default=False, help="Fill trace automatically") + + args, unknown = parser.parse_known_args() + + sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n") + + pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) + main(pkl_list_main, name_list_main, autofill=args.autofill, title=args.title, log=not args.log, save=args.save, + y_min=args.min, y_max=args.max) diff --git a/HPOlib/Plotting/plotParam.py b/HPOlib/Plotting/plotParam.py index 44b74971..d660cb47 100644 --- a/HPOlib/Plotting/plotParam.py +++ b/HPOlib/Plotting/plotParam.py @@ -129,29 +129,11 @@ def plot_params(value_list, result_list, name, save="", title="", jitter=0): show() -def main(): - prog = "python plot_param.py WhatIsThis * " - description = "Plot one param wrt to minfunction value" - - parser = ArgumentParser(description=description, prog=prog) - parser.add_argument("-s", "--save", dest="save", default="", - help="Where to save plot instead of showing it?") - parser.add_argument("-p", "--parameter", dest="param", default="", - help="Which parameter to plot") - parser.add_argument("-j", "--jitter", dest="jitter", default=0, type=float, - help="Jitter data?") - parser.add_argument("-t", "--title", dest="title", default="", - help="Choose a supertitle for the plot") - - args, unknown = parser.parse_known_args() - - sys.stdout.write("\nFound " + str(len(unknown)) + " argument[s]...\n") - pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) - +def main(pkl_list, name_list, param, save="", title="", jitter=0): if len(name_list) > 1: raise NotImplementedError("No support for more than one experiment") - param = "-" + args.param + mod_param = "-" + param value_list = list() result_list = list() @@ -161,21 +143,39 @@ def main(): trials = cPickle.load(fh) fh.close() for t in trials["trials"]: - if param in t["params"]: - k, value = translate_para(args.param, t["params"][param]) + if mod_param in t["params"]: + k, value = translate_para(param, t["params"][mod_param]) value_list.append(float(value.strip("'"))) result_list.append(t["result"]) param_set.update(t["params"].keys()) if len(value_list) == 0: print("No values found for param '%s', Available params:\n%s" % - (args.param, "\n".join([p[1:] for p in param_set]))) + (param, "\n".join([p[1:] for p in param_set]))) sys.exit(1) else: - print "Found %s values for %s" % (str(len(value_list)), args.param) + print "Found %s values for %s" % (str(len(value_list)), param) - plot_params(value_list=value_list, result_list=result_list, name=args.param, save=args.save, title=args.title, - jitter=args.jitter) + plot_params(value_list=value_list, result_list=result_list, name=param, save=save, title=title, + jitter=jitter) if __name__ == "__main__": - main() + prog = "python plot_param.py WhatIsThis * " + description = "Plot one param wrt to minfunction value" + + parser = ArgumentParser(description=description, prog=prog) + parser.add_argument("-s", "--save", dest="save", default="", + help="Where to save plot instead of showing it?") + parser.add_argument("-p", "--parameter", dest="param", default="", + help="Which parameter to plot") + parser.add_argument("-j", "--jitter", dest="jitter", default=0, type=float, + help="Jitter data?") + parser.add_argument("-t", "--title", dest="title", default="", + help="Choose a supertitle for the plot") + + args, unknown = parser.parse_known_args() + + sys.stdout.write("\nFound " + str(len(unknown)) + " argument[s]...\n") + pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) + main(pkl_list=pkl_list_main, name_list=name_list_main, param=args.param, + save=args.save, title=args.title, jitter=args.jitter) diff --git a/HPOlib/Plotting/plotTrace.py b/HPOlib/Plotting/plotTrace.py index 3517e716..392564c6 100644 --- a/HPOlib/Plotting/plotTrace.py +++ b/HPOlib/Plotting/plotTrace.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + ## # wrapping: A program making it easy to use hyperparameter # optimization software. @@ -16,8 +18,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python - from argparse import ArgumentParser import itertools import sys @@ -116,7 +116,28 @@ def plot_optimization_trace(trial_list, name_list, optimum=0, title="", log=Fals show() -def main(): +def main(pkl_list, name_list, optimum=0, title="", log=False, save="", y_max=0, y_min=0): + + trial_list = list() + for i in range(len(pkl_list)): + if len(pkl_list[i]) != 1: + raise ValueError("%s is more than !" % str(pkl_list)) + fh = open(pkl_list[i][0]) + trl = cPickle.load(fh) + fh.close() + trial_list.append(plot_util.extract_trials(trl)) + + sys.stdout.write("Plotting trace\n") + plot_optimization_trace(trial_list=trial_list, name_list=name_list, optimum=optimum, + title=title, log=not log, + save=save, y_max=y_max, y_min=y_min) + + if args.save != "": + sys.stdout.write("Saved plot to " + args.save + "\n") + else: + sys.stdout.write("..Done\n") + +if __name__ == "__main__": prog = "python plotTrace.py WhatIsThis [WhatIsThis ]" description = "Plot a Trace with evaluated points wrt to performance" @@ -139,30 +160,9 @@ def main(): args, unknown = parser.parse_known_args() - optimum = args.optimum - sys.stdout.write("Found " + str(len(unknown)) + " arguments\n") - pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) + pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) - trial_list = list() - for i in range(len(pkl_list)): - if len(pkl_list[i]) != 1: - raise ValueError("%s is more than !" % str(pkl_list)) - fh = open(pkl_list[i][0]) - trl = cPickle.load(fh) - fh.close() - trial_list.append(plot_util.extract_trials(trl)) - - sys.stdout.write("Plotting trace\n") - plot_optimization_trace(trial_list=trial_list, name_list=name_list, optimum=optimum, - title=args.title, log=not args.log, - save=args.save, y_max=args.max, y_min=args.min) - - if args.save != "": - sys.stdout.write("Saved plot to " + args.save + "\n") - else: - sys.stdout.write("..Done\n") - -if __name__ == "__main__": - main() + main(pkl_list=pkl_list_main, name_list=name_list_main, optimum=args.optimum, + title=args.title, log=args.log, save=args.save, y_max=args.max, y_min=args.min) diff --git a/HPOlib/Plotting/plotTraceWithStd_perEval.py b/HPOlib/Plotting/plotTraceWithStd_perEval.py index 2e5acee4..4229047d 100644 --- a/HPOlib/Plotting/plotTraceWithStd_perEval.py +++ b/HPOlib/Plotting/plotTraceWithStd_perEval.py @@ -117,7 +117,39 @@ def plot_optimization_trace(trial_list, name_list, optimum=0, title="", show() -def main(): +def main(pkl_list, name_list, autofill, optimum=0, save="", title="", log=False, y_min=0, y_max=0, scale_std=1): + + trial_list = list() + for i in range(len(pkl_list)): + trial_list.append(list()) + for pkl in pkl_list[i]: + fh = open(pkl, "r") + trials = cPickle.load(fh) + fh.close() + + trace = plot_util.extract_trajectory(trials) + trial_list[-1].append(np.array(trace)) + + for i in range(len(trial_list)): + max_len = max([len(ls) for ls in trial_list[i]]) + for t in range(len(trial_list[i])): + if len(trial_list[i][t]) < max_len and autofill: + diff = max_len - len(trial_list[i][t]) + # noinspection PyUnusedLocal + trial_list[i][t] = np.append(trial_list[i][t], [trial_list[i][t][-1] for x in range(diff)]) + elif len(trial_list[i][t]) < max_len and not autofill: + raise ValueError("(%s != %s), Traces do not have the same length, please use -a" % + (str(max_len), str(len(trial_list[i][t])))) + + plot_optimization_trace(trial_list, name_list, optimum, title=title, log=not log, + save=save, y_min=y_min, y_max=y_max, scale_std=scale) + + if save != "": + sys.stdout.write("Saved plot to " + save + "\n") + else: + sys.stdout.write("..Done\n") + +if __name__ == "__main__": prog = "python plotTraceWithStd.py WhatIsThis [WhatIsThis ]" description = "Plot a Trace with std for multiple experiments" @@ -146,44 +178,9 @@ def main(): args, unknown = parser.parse_known_args() - sys.stdout.write("Generate plot for ") - optimum = args.optimum - title = args.title - sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n") - trial_list = list() - - pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) - - for i in range(len(pkl_list)): - trial_list.append(list()) - for pkl in pkl_list[i]: - fh = open(pkl, "r") - trials = cPickle.load(fh) - fh.close() - - trace = plot_util.extract_trajectory(trials) - trial_list[-1].append(np.array(trace)) + pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) - for i in range(len(trial_list)): - max_len = max([len(ls) for ls in trial_list[i]]) - for t in range(len(trial_list[i])): - if len(trial_list[i][t]) < max_len and args.autofill: - diff = max_len - len(trial_list[i][t]) - # noinspection PyUnusedLocal - trial_list[i][t] = np.append(trial_list[i][t], [trial_list[i][t][-1] for x in range(diff)]) - elif len(trial_list[i][t]) < max_len and not args.autofill: - raise ValueError("(%s != %s), Traces do not have the same length, please use -a" % - (str(max_len), str(len(trial_list[i][t])))) - - plot_optimization_trace(trial_list, name_list, optimum, title=title, log=not args.log, - save=args.save, y_min=args.min, y_max=args.max, scale_std=args.scale) - - if args.save != "": - sys.stdout.write("Saved plot to " + args.save + "\n") - else: - sys.stdout.write("..Done\n") - -if __name__ == "__main__": - main() + main(pkl_list=pkl_list_main, name_list=name_list_main, autofill=args.autofill, optimum=args.optimum, + save=args.save, title=args.title, log=args.log, y_min=args.min, y_max=args.max, scale_std=args.scale) diff --git a/HPOlib/Plotting/plotTraceWithStd_perTime.py b/HPOlib/Plotting/plotTraceWithStd_perTime.py index 638af260..5ac95490 100644 --- a/HPOlib/Plotting/plotTraceWithStd_perTime.py +++ b/HPOlib/Plotting/plotTraceWithStd_perTime.py @@ -178,46 +178,15 @@ def fill_trajectories(trace_list, times_list): return trajectories, times -def main(): - prog = "python plotTraceWithStd.py WhatIsThis [WhatIsThis ]" - description = "Plot a Trace with std for multiple experiments" - - parser = ArgumentParser(description=description, prog=prog) - - # Options for specific benchmarks - parser.add_argument("-o", "--optimum", type=float, dest="optimum", - default=0, help="If not set, the optimum is supposed to be zero") - - # Options which are available only for this plot - parser.add_argument("-a", "--autofill", action="store_true", dest="autofill", - default=False, help="Fill trace automatically") - parser.add_argument("-c", "--scale", type=float, dest="scale", - default=1, help="Multiply std to get a nicer plot") - # General Options - parser.add_argument("-l", "--nolog", action="store_true", dest="log", - default=False, help="Do NOT plot on log scale") - parser.add_argument("--max", dest="max", type=float, - default=0, help="Maximum of the plot") - parser.add_argument("--min", dest="min", type=float, - default=0, help="Minimum of the plot") - parser.add_argument("-s", "--save", dest="save", - default="", help="Where to save plot instead of showing it?") - parser.add_argument("-t", "--title", dest="title", - default="", help="Optional supertitle for plot") - - args, unknown = parser.parse_known_args() +def main(pkl_list, name_list, autofill, optimum=0, save="", title="", log=False, y_min=0, y_max=0, scale_std=1): sys.stdout.write("Generate plot for ") optimum = args.optimum title = args.title - sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n") - trial_list = list() times_list = list() - pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) - for i in range(len(pkl_list)): tmp_trial_list = list() tmp_times_list = list() @@ -238,20 +207,53 @@ def main(): for i in range(len(trial_list)): max_len = max([len(ls) for ls in trial_list[i]]) for t in range(len(trial_list[i])): - if len(trial_list[i][t]) < max_len and args.autofill: + if len(trial_list[i][t]) < max_len and autofill: diff = max_len - len(trial_list[i][t]) # noinspection PyUnusedLocal trial_list[i][t] = np.append(trial_list[i][t], [trial_list[i][t][-1] for x in range(diff)]) - elif len(trial_list[i][t]) < max_len and not args.autofill: + elif len(trial_list[i][t]) < max_len and not autofill: raise ValueError("(%s != %s), Traces do not have the same length, please use -a" % (str(max_len), str(len(trial_list[i][t])))) - plot_optimization_trace(trial_list, name_list, times_list, optimum, title=title, log=not args.log, - save=args.save, y_min=args.min, y_max=args.max, scale_std=args.scale) + plot_optimization_trace(trial_list, name_list, times_list, optimum, title=title, log=not log, + save=save, y_min=y_min, y_max=y_max, scale_std=scale_std) if args.save != "": sys.stdout.write("Saved plot to " + args.save + "\n") else: sys.stdout.write("..Done\n") if __name__ == "__main__": - main() + prog = "python plotTraceWithStd.py WhatIsThis [WhatIsThis ]" + description = "Plot a Trace with std for multiple experiments" + + parser = ArgumentParser(description=description, prog=prog) + + # Options for specific benchmarks + parser.add_argument("-o", "--optimum", type=float, dest="optimum", + default=0, help="If not set, the optimum is supposed to be zero") + + # Options which are available only for this plot + parser.add_argument("-a", "--autofill", action="store_true", dest="autofill", + default=False, help="Fill trace automatically") + parser.add_argument("-c", "--scale", type=float, dest="scale", + default=1, help="Multiply std to get a nicer plot") + # General Options + parser.add_argument("-l", "--nolog", action="store_true", dest="log", + default=False, help="Do NOT plot on log scale") + parser.add_argument("--max", dest="max", type=float, + default=0, help="Maximum of the plot") + parser.add_argument("--min", dest="min", type=float, + default=0, help="Minimum of the plot") + parser.add_argument("-s", "--save", dest="save", + default="", help="Where to save plot instead of showing it?") + parser.add_argument("-t", "--title", dest="title", + default="", help="Optional supertitle for plot") + + args, unknown = parser.parse_known_args() + + sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n") + + pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) + + main(pkl_list_main, name_list_main, autofill=args.autofill, optimum=args.optimum, save=args.save, + title=args.title, log=args.log, y_min=args.min, y_max=args.max, scale_std=args.scale) diff --git a/HPOlib/Plotting/statistics.py b/HPOlib/Plotting/statistics.py index 5a5830c9..09004391 100644 --- a/HPOlib/Plotting/statistics.py +++ b/HPOlib/Plotting/statistics.py @@ -114,22 +114,12 @@ def _mann_whitney_u(x, y=None): """ -def main(): - prog = "python statistics.py WhatIsThis WhatIsThis [WhatIsThis ]" - description = "Return some statistical information" - - parser = ArgumentParser(description=description, prog=prog) - - parser.add_argument("-c", "--c", dest="cut", default=False, - type=int, help="Only consider that many evaluations") - args, unknown = parser.parse_known_args() +def main(pkl_list, name_list): best_dict = dict() idx_dict = dict() keys = list() - pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) - for i in range(len(name_list)): keys.append(name_list[i][0]) best_dict[name_list[i][0]] = list() @@ -195,4 +185,14 @@ def main(): sys.stdout.write("------------------------------------------------------------------------\n") if __name__ == "__main__": - main() + prog = "python statistics.py WhatIsThis WhatIsThis [WhatIsThis ]" + description = "Return some statistical information" + + parser = ArgumentParser(description=description, prog=prog) + + parser.add_argument("-c", "--c", dest="cut", default=False, + type=int, help="Only consider that many evaluations") + args, unknown = parser.parse_known_args() + + pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) + main(pkl_list=pkl_list_main, name_list=name_list_main) From 8a021a95280f3c0abd9d00de4254b0e2cbb02bc3 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 3 Mar 2014 15:47:39 +0100 Subject: [PATCH 050/139] REFACTOR #1: one plot script doAllPlots.py is a python script that works like a regular plot script and accepts arguments python doAllPlots.py NAME [-s ] [-l] python doAllPlots.py NAME [-s ] [-l] python doAllPlots.py NAME NAME1 [-s ] [-l] Based on the given arguments it decides which plots perform So far it performs 1 Name with 1 pkl: plotTrace.py >1 Name: plotBoxWhisker.py statistics.py always: plotOptimizerOverhead.py TraceWithStd_perEval.py TraceWithStd_perTime.py --- HPOlib/Plotting/doAllPlots.py | 229 ++++++++++++++++++++ HPOlib/Plotting/doAllPlots.sh | 209 ------------------ HPOlib/Plotting/plotTrace.py | 4 +- HPOlib/Plotting/plotTraceWithStd_perEval.py | 2 +- HPOlib/Plotting/plotTraceWithStd_perTime.py | 8 +- 5 files changed, 234 insertions(+), 218 deletions(-) create mode 100644 HPOlib/Plotting/doAllPlots.py delete mode 100644 HPOlib/Plotting/doAllPlots.sh diff --git a/HPOlib/Plotting/doAllPlots.py b/HPOlib/Plotting/doAllPlots.py new file mode 100644 index 00000000..887a7861 --- /dev/null +++ b/HPOlib/Plotting/doAllPlots.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python + +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from argparse import ArgumentParser +import os +import subprocess +import sys +import time + +from HPOlib.Plotting import plot_util +from HPOlib.wrapping_util import format_traceback + + +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" + + +def _plot_trace(pkl_list, name_list, save="", log=False): + # We have one pkl per experiment + + plotting_dir = os.path.dirname(os.path.realpath(__file__)) + cur_dir = os.getcwd() + + # noinspection PyBroadException + try: + os.chdir(plotting_dir) + import plotTrace + plotTrace.main(pkl_list, name_list, save=save, log=log) + os.chdir(cur_dir) + sys.stdout.write("passed\n") + except Exception, e: + sys.stderr.write(format_traceback(sys.exc_info())) + sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) + + +def _trace_with_std_per_eval(pkl_list, name_list, save="", log=False): + plotting_dir = os.path.dirname(os.path.realpath(__file__)) + cur_dir = os.getcwd() + + # noinspection PyBroadException + try: + os.chdir(plotting_dir) + import plotTraceWithStd_perEval + plotTraceWithStd_perEval.main(pkl_list, name_list, autofill=True, + optimum=0, save=save, log=log) + os.chdir(cur_dir) + sys.stdout.write("passed\n") + except Exception, e: + sys.stderr.write(format_traceback(sys.exc_info())) + sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) + + +def _trace_with_std_per_time(pkl_list, name_list, save="", log=False): + plotting_dir = os.path.dirname(os.path.realpath(__file__)) + cur_dir = os.getcwd() + + # noinspection PyBroadException + try: + os.chdir(plotting_dir) + import plotTraceWithStd_perTime + plotTraceWithStd_perTime.main(pkl_list, name_list, autofill=True, + optimum=0, save=save, log=log) + os.chdir(cur_dir) + sys.stdout.write("passed\n") + except Exception, e: + sys.stderr.write(format_traceback(sys.exc_info())) + sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) + + +def _optimizer_overhead(pkl_list, name_list, save, log): + plotting_dir = os.path.dirname(os.path.realpath(__file__)) + cur_dir = os.getcwd() + + # noinspection PyBroadException + try: + os.chdir(plotting_dir) + import plotOptimizerOverhead + plotOptimizerOverhead.main(pkl_list, name_list, autofill=True, + log=log, save=save) + os.chdir(cur_dir) + sys.stdout.write("passed\n") + except Exception, e: + sys.stderr.write(format_traceback(sys.exc_info())) + sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) + + +def _box_whisker(pkl_list, name_list, save): + plotting_dir = os.path.dirname(os.path.realpath(__file__)) + cur_dir = os.getcwd() + + # noinspection PyBroadException + try: + os.chdir(plotting_dir) + import plotBoxWhisker + plotBoxWhisker.main(pkl_list, name_list, save=save) + os.chdir(cur_dir) + sys.stdout.write("passed\n") + except Exception, e: + sys.stderr.write(format_traceback(sys.exc_info())) + sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) + + +def _statistics(pkl_list, name_list, save=""): + plotting_dir = os.path.dirname(os.path.realpath(__file__)) + cur_dir = os.getcwd() + + # noinspection PyBroadException + try: + os.chdir(plotting_dir) + cmd = ["python statistics.py"] + for i in range(len(name_list)): + cmd.append(name_list[i][0]) + for pkl in pkl_list[i]: + cmd.append(pkl) + if save is not "": + fh = open(save, "w") + subprocess.check_call(" ".join(cmd), shell=True, stdin=fh, stdout=fh, stderr=fh) + fh.close() + else: + proc = subprocess.Popen(" ".join(cmd), shell=True, stdout=subprocess.PIPE) + out = proc.communicate()[0] + #print the output of the child process to stdout + print (out) + os.chdir(cur_dir) + sys.stdout.write("passed\n") + except Exception, e: + sys.stderr.write(format_traceback(sys.exc_info())) + sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) + + +def main(pkl_list, name_list, save_dir="", log=""): + time_str = int(time.time() % 1000) + + if not os.path.isdir(save_dir) and save_dir is not "": + os.mkdir(save_dir) + + for i in range(len(pkl_list)): + for j in range(len(pkl_list[i])): + if os.path.exists(pkl_list[i][j]): + pkl_list[i][j] = os.path.abspath(pkl_list[i][j]) + else: + raise NotImplementedError("%s is not a valid file" % pkl_list[i][j]) + + if len(name_list) == 1 and name_list[0][1] == 1: + # We have one exp and one pkl + if save_dir is not "": + tmp_save = os.path.join(save_dir, "plotTrace_%s.png" % time_str) + else: + tmp_save = save_dir + sys.stdout.write("plotTrace.py ... %s ..." % tmp_save) + _plot_trace(pkl_list=pkl_list, name_list=name_list, save=tmp_save, log=log) + + if len(name_list) > 1: + # Some plots only make sense, if there are many experiments + # BoxWhisker + if save_dir is not "": + tmp_save = os.path.join(save_dir, "BoxWhisker_%s.png" % time_str) + else: + tmp_save = save_dir + sys.stdout.write("plotBoxWhisker.py ... %s ..." % tmp_save) + _box_whisker(pkl_list=pkl_list, name_list=name_list, save=tmp_save) + + # statistics + if save_dir is not "": + tmp_save = os.path.join(save_dir, "statistics_%s.txt" % time_str) + else: + tmp_save = save_dir + sys.stdout.write("statistics.py ... %s ..." % tmp_save) + _statistics(pkl_list=pkl_list, name_list=name_list, save=tmp_save) + + # We can always plot this + # OptimizerOverhead + if save_dir is not "": + tmp_save = os.path.join(save_dir, "OptimizerOverhead_%s.png" % time_str) + else: + tmp_save = save_dir + sys.stdout.write("plotOptimizerOverhead.py ... %s ..." % tmp_save) + _optimizer_overhead(pkl_list=pkl_list, name_list=name_list, save=tmp_save, log=log) + + # Error Trace with Std + if save_dir is not "": + tmp_save = os.path.join(save_dir, "TraceWithStd_perEval_%s.png" % time_str) + else: + tmp_save = save_dir + sys.stdout.write("TraceWithStd_perEval.py ... %s ..." % tmp_save) + _trace_with_std_per_eval(pkl_list=pkl_list, name_list=name_list, save=tmp_save, log=log) + + if save_dir is not "": + tmp_save = os.path.join(save_dir, "TraceWithStd_perTime_%s.png" % time_str) + else: + tmp_save = save_dir + sys.stdout.write("TraceWithStd_perTime.py ... %s ..." % tmp_save) + _trace_with_std_per_time(pkl_list=pkl_list, name_list=name_list, save=tmp_save, log=log) + +if __name__ == "__main__": + prog = "python doAllPlots.py WhatIsThis [WhatIsThis ]" + description = "Tries to save as many plots as possible" + + parser = ArgumentParser(description=description, prog=prog) + + # General Options + parser.add_argument("-l", "--nolog", action="store_true", dest="log", + default=False, help="Do NOT plot on log scale") + parser.add_argument("-s", "--save", dest="save", + default="", help="Where to save plots? (directory)") + + args, unknown = parser.parse_known_args() + + sys.stdout.write("Found " + str(len(unknown)) + " arguments\n") + + pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) + main(pkl_list=pkl_list_main, name_list=name_list_main, log=args.log, save_dir=args.save) diff --git a/HPOlib/Plotting/doAllPlots.sh b/HPOlib/Plotting/doAllPlots.sh deleted file mode 100644 index 4a868be1..00000000 --- a/HPOlib/Plotting/doAllPlots.sh +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env bash - -# Authors: Katharina Eggensperger and Matthias Feurer -# License: 3-clause BSD License -# Contact: "automl.org" - -#minTimeTrace=0 -#maxTimeTrace=0 -#minErrorTrace=0 -#maxErrorTrace=0 -#minTrace=0 -#maxTrace=0 -#minBox=0 -#maxBox=0 -#specArgs="" - -#LDA -#minTimeTrace=-1 -#maxTimeTrace=16 -#minErrorTrace=1260 -#maxErrorTrace=1350 -#minTrace=1000 -#maxTrace=4000 -#minBox=1260 -#maxBox=1350 -#specArgs="" - -#SVM -#minTimeTrace=-1 -#maxTimeTrace=12 -#minErrorTrace=0.24 -#maxErrorTrace=0.26 -#minTrace=0.24 -#maxTrace=0.3 -#minBox=0.24 -#maxBox=0.26 -#specArgs="" - -#Har6 -#minTimeTrace=-1 -#maxTimeTrace=12 -#minErrorTrace=0 -#maxErrorTrace=4 -#minTrace=0 -#maxTrace=4 -#minBox=0 -#maxBox=4 -#specArgs=" -j" - -#Branin -#minTimeTrace=-1 -#maxTimeTrace=12 -#minErrorTrace=0 -#maxErrorTrace=1.5 -#minTrace=0 -#maxTrace=2 -#minBox=0 -#maxBox=2 -#specArgs=" -b" - - -#Net convex -#minTimeTrace=-1 -#maxTimeTrace=12 -#minErrorTrace=0.15 -#maxErrorTrace=0.5 -#minTrace=0.15 -#maxTrace=0.5 -#minBox=0.15 -#maxBox=0.3 -#specArgs="" - -#Net MRBI -minTimeTrace=-1 -maxTimeTrace=12 -minErrorTrace=0.45 -maxErrorTrace=1 -minTrace=0.45 -maxTrace=1 -minBox=0.45 -maxBox=0.7 -specArgs="" - -#AutoWEKA -#minTimeTrace=-1 -#maxTimeTrace=12 -#minErrorTrace=20 -#maxErrorTrace=50 -#minTrace=20 -#maxTrace=50 -#minBox=20 -#maxBox=50 -#specArgs="" - -#Logistic Regression -#minTimeTrace=0 -#maxTimeTrace=0 -#minErrorTrace=0.06 -#maxErrorTrace=0.26 -#minTrace=0.08 -#maxTrace=0.28 -#minBox=0.08 -#maxBox=0.12 -#specArgs="" - -echo $1 - -smac="smac $1/smac_*_*/smac.pkl" -tpe="tpe $1/tpe_*_*/tpe.pkl" -spearmint="spearmint $1/spearmint_*_*/spearmint.pkl" -random="random $1/random_*_*/random.pkl" - -#smac="smac $1/smac_*.pkl" -#tpe="tpe $1/tpe_*.pkl" -#spearmint="spearmint $1/spearmint_*.pkl" -#random="random $1/random_*.pkl" - -# build arguments -argu="" -if [ -d "$1/smac" ] -then - argu="$argu $smac" -fi - -if [ -d "$1/tpe" ] -then - argu="$argu $tpe" -fi - -if [ -d "$1/spearmint" ] -then - argu="$argu $spearmint" -fi - -if [ -d "$1/random" ] -then - argu="$argu $random" -fi - -output="$1/Evaluation" -mkdir $output -echo "#########################################################################" -echo "Generating basic statistics" -python statistics.py $argu > "$output/statistics.txt" - -echo "#########################################################################" -echo "Gather validation and test results" - -now=`pwd` -cd $1 -echo `LANG=en_us_8859_1 bash "$now/results.sh" .` > "$output/results.txt" -cd $now - -echo "#########################################################################" -echo "Plotting the time trace of the optimizer" -python plotTimeTrace.py $argu -s "$output/TimeTraceLog.png" --min $minTimeTrace --max $maxTimeTrace -python plotTimeTrace.py $argu -s "$output/TimeTraceLog.pdf" --min $minTimeTrace --max $maxTimeTrace -python plotTimeTrace.py $argu -l -s "$output/TimeTrace.png" --min $minTimeTrace --max $maxTimeTrace -python plotTimeTrace.py $argu -l -s "$output/TimeTrace.pdf" --min $minTimeTrace --max $maxTimeTrace - -echo "#########################################################################" -echo "Plot the error trace with standard deviation" -python plotTraceWithStd.py $argu -a -s "$output/ErrorTraceLog.png" --min $minErrorTrace --max $maxErrorTrace $specArgs -python plotTraceWithStd.py $argu -a -s "$output/ErrorTraceLog.pdf" --min $minErrorTrace --max $maxErrorTrace $specArgs -python plotTraceWithStd.py $argu --nolog -a -s "$output/ErrorTrace.png" --min $minErrorTrace --max $maxErrorTrace $specArgs -python plotTraceWithStd.py $argu --nolog -a -s "$output/ErrorTrace.pdf" --min $minErrorTrace --max $maxErrorTrace $specArgs - -echo "#########################################################################" -echo "Plot the distribution of hyperparameters together with their response value" -mkdir $output/Params -python plot.py $tpe -s "$output/Params/TPE_para_" -python plot.py $smac -s "$output/Params/SMAC_para_" -python plot.py $spearmint -s "$output/Params/Spearmint_para_" -python plot.py MIX $1/smac_*_*/smac.pkl $1/tpe_*_*/tpe.pkl $1/spearmint_*_*/spearmint.pkl -s $output/Params/MIX_para_ - -echo "#########################################################################" -echo "Plot an optimization trace for every single optimization" -idx=0 -opt="" - -for pkl in $argu -do - if [ -f $pkl ] - then - opt=`echo $pkl | rev| cut -d"/" -f1 | rev | cut -d"." -f1` - #python plotTrace.py $opt $pkl -s "$output/Trace_${opt}_$idx.pdf" --min $minTrace --max $maxTrace $specArgs - #python plotTrace.py $opt $pkl -s "$output/Trace_${opt}_$idx.png" --min $minTrace --max $maxTrace $specArgs - fi - idx=$((idx+1)) -done - -for i in `seq 1000 1000 10000` -do - python plotTrace.py smac $1/smac_${i}_*/smac.pkl tpe $1/tpe_${i}_*/tpe.pkl spearmint $1/spearmint_${i}_*/spearmint.pkl -s "$output/Trace_all_$i.png" --min $minTrace --max $maxTrace $specArgs - python plotTrace.py smac $1/smac_${i}_*/smac.pkl tpe $1/tpe_${i}_*/tpe.pkl spearmint $1/spearmint_${i}_*/spearmint.pkl -s "$output/Trace_all_$i.pdf" --min $minTrace --max $maxTrace $specArgs - #python plotTrace.py smac $1/smac_${i}.pkl tpe $1/tpe_${i}.pkl spearmint $1/spearmint_${i}.pkl -s "$output/Trace_all_$i.png" --min $minTrace --max $maxTrace $specArgs - #python plotTrace.py smac $1/smac_${i}.pkl tpe $1/tpe_${i}.pkl spearmint $1/spearmint_${i}.pkl -s "$output/Trace_all_$i.pdf" --min $minTrace --max $maxTrace $specArgs -done - -echo "#########################################################################" -echo "Plot a box-whisker plot" -python plotBoxWhisker.py $argu -s "$output/BoxWhisker.png" --min $minBox --max $maxBox -python plotBoxWhisker.py $argu -s "$output/BoxWhisker.pdf" --min $minBox --max $maxBox - - - - - - diff --git a/HPOlib/Plotting/plotTrace.py b/HPOlib/Plotting/plotTrace.py index 392564c6..94fb2ee9 100644 --- a/HPOlib/Plotting/plotTrace.py +++ b/HPOlib/Plotting/plotTrace.py @@ -132,8 +132,8 @@ def main(pkl_list, name_list, optimum=0, title="", log=False, save="", y_max=0, title=title, log=not log, save=save, y_max=y_max, y_min=y_min) - if args.save != "": - sys.stdout.write("Saved plot to " + args.save + "\n") + if save != "": + sys.stdout.write("Saved plot to " + save + "\n") else: sys.stdout.write("..Done\n") diff --git a/HPOlib/Plotting/plotTraceWithStd_perEval.py b/HPOlib/Plotting/plotTraceWithStd_perEval.py index 4229047d..937a3ef0 100644 --- a/HPOlib/Plotting/plotTraceWithStd_perEval.py +++ b/HPOlib/Plotting/plotTraceWithStd_perEval.py @@ -142,7 +142,7 @@ def main(pkl_list, name_list, autofill, optimum=0, save="", title="", log=False, (str(max_len), str(len(trial_list[i][t])))) plot_optimization_trace(trial_list, name_list, optimum, title=title, log=not log, - save=save, y_min=y_min, y_max=y_max, scale_std=scale) + save=save, y_min=y_min, y_max=y_max, scale_std=scale_std) if save != "": sys.stdout.write("Saved plot to " + save + "\n") diff --git a/HPOlib/Plotting/plotTraceWithStd_perTime.py b/HPOlib/Plotting/plotTraceWithStd_perTime.py index 5ac95490..6d2de297 100644 --- a/HPOlib/Plotting/plotTraceWithStd_perTime.py +++ b/HPOlib/Plotting/plotTraceWithStd_perTime.py @@ -180,10 +180,6 @@ def fill_trajectories(trace_list, times_list): def main(pkl_list, name_list, autofill, optimum=0, save="", title="", log=False, y_min=0, y_max=0, scale_std=1): - sys.stdout.write("Generate plot for ") - optimum = args.optimum - title = args.title - trial_list = list() times_list = list() @@ -217,8 +213,8 @@ def main(pkl_list, name_list, autofill, optimum=0, save="", title="", log=False, plot_optimization_trace(trial_list, name_list, times_list, optimum, title=title, log=not log, save=save, y_min=y_min, y_max=y_max, scale_std=scale_std) - if args.save != "": - sys.stdout.write("Saved plot to " + args.save + "\n") + if save != "": + sys.stdout.write("Saved plot to " + save + "\n") else: sys.stdout.write("..Done\n") From e57a956566a1ce16ef171205312c88eba50784fc Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 3 Mar 2014 16:29:02 +0100 Subject: [PATCH 051/139] REFACTOR #1: One scrip to rule them all: HPOlib-plot HPOlib-plot calls HPOlib/Plotting/doAllPlots.py and will be installed with setup.py. Fixes #1 --- HPOlib/Plotting/doAllPlots.py | 40 ++++++++++++++++++-------------- HPOlib-run => scripts/HPOlib-run | 0 setup.py | 8 ++++--- 3 files changed, 27 insertions(+), 21 deletions(-) rename HPOlib-run => scripts/HPOlib-run (100%) diff --git a/HPOlib/Plotting/doAllPlots.py b/HPOlib/Plotting/doAllPlots.py index 887a7861..85c9cc7a 100644 --- a/HPOlib/Plotting/doAllPlots.py +++ b/HPOlib/Plotting/doAllPlots.py @@ -145,7 +145,27 @@ def _statistics(pkl_list, name_list, save=""): sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) -def main(pkl_list, name_list, save_dir="", log=""): +def main(): + prog = "python doAllPlots.py WhatIsThis [WhatIsThis ]" + description = "Tries to save as many plots as possible" + + parser = ArgumentParser(description=description, prog=prog) + + # General Options + parser.add_argument("-l", "--nolog", action="store_true", dest="log", + default=False, help="Do NOT plot on log scale") + parser.add_argument("-s", "--save", dest="save", + default="", help="Where to save plots? (directory)") + + args, unknown = parser.parse_known_args() + + sys.stdout.write("Found " + str(len(unknown)) + " arguments\n") + + save_dir = args.save + log = args.log + + pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) + time_str = int(time.time() % 1000) if not os.path.isdir(save_dir) and save_dir is not "": @@ -210,20 +230,4 @@ def main(pkl_list, name_list, save_dir="", log=""): _trace_with_std_per_time(pkl_list=pkl_list, name_list=name_list, save=tmp_save, log=log) if __name__ == "__main__": - prog = "python doAllPlots.py WhatIsThis [WhatIsThis ]" - description = "Tries to save as many plots as possible" - - parser = ArgumentParser(description=description, prog=prog) - - # General Options - parser.add_argument("-l", "--nolog", action="store_true", dest="log", - default=False, help="Do NOT plot on log scale") - parser.add_argument("-s", "--save", dest="save", - default="", help="Where to save plots? (directory)") - - args, unknown = parser.parse_known_args() - - sys.stdout.write("Found " + str(len(unknown)) + " arguments\n") - - pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) - main(pkl_list=pkl_list_main, name_list=name_list_main, log=args.log, save_dir=args.save) + main() \ No newline at end of file diff --git a/HPOlib-run b/scripts/HPOlib-run similarity index 100% rename from HPOlib-run rename to scripts/HPOlib-run diff --git a/setup.py b/setup.py index a64997eb..c037fc32 100644 --- a/setup.py +++ b/setup.py @@ -20,13 +20,14 @@ 'HPOlib.benchmarks': 'HPOlib/benchmarks', 'HPOlib.optimizers': 'HPOlib/optimizers', 'HPOlib.benchmarks.branin': 'HPOlib/benchmarks/branin', - 'HPOlib.benchmarks.har6': 'HPOlib/benchmarks/har6'} + 'HPOlib.benchmarks.har6': 'HPOlib/benchmarks/har6', + 'HPOlib.Plotting': 'HPOlib/Plotting'} package_data = {'HPOlib': ['*/params.txt', '*/space.py', '*/config.pb'], 'HPOlib.benchmarks.branin': ['*.cfg', '*/params.txt', '*/space.py', '*/config.pb'], 'HPOlib.benchmarks.har6': ['*.cfg', '*/params.txt', '*/space.py', '*/config.pb'], 'HPOlib.config_parser': ['*.cfg']} data_files = [] -scripts = ['HPOlib-run', 'runsolver/src/runsolver'] +scripts = ['scripts/HPOlib-run', 'scripts/HPOlib-plot', 'runsolver/src/runsolver'] def read(fname): @@ -39,7 +40,8 @@ def get_find_packages(): 'HPOlib.benchmarks', 'HPOlib.optimizers', 'HPOlib.benchmarks.branin', - 'HPOlib.benchmarks.har6'] + 'HPOlib.benchmarks.har6', + 'HPOlib.Plotting'] return packages From ed6c01dd21b14864a2a0dd6540e93b9aa1a6b520 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 3 Mar 2014 16:30:35 +0100 Subject: [PATCH 052/139] REFACTOR #1: One script to rule them all: HPOlib-plot --- HPOlib/wrapping.py | 1 + scripts/HPOlib-plot | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 scripts/HPOlib-plot diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 0e2cc349..2a8c0d92 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -356,6 +356,7 @@ def main(): ret = proc.returncode logger.info("\n-----------------------END--------------------------------------") + fh.close() trials = Experiment.Experiment(optimizer_dir_in_experiment, optimizer) trials.endtime.append(time.time()) diff --git a/scripts/HPOlib-plot b/scripts/HPOlib-plot new file mode 100644 index 00000000..3b1aecba --- /dev/null +++ b/scripts/HPOlib-plot @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" + +from HPOlib.Plotting import doAllPlots + +doAllPlots.main() \ No newline at end of file From 4a59d0ac99b4c484f9ff27f26bd1b602da5584f2 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 3 Mar 2014 16:53:48 +0100 Subject: [PATCH 053/139] FIX #25: add sys.exit(1) When path to optimizer (dir which is specified in *Default.cfg) not exists, optimizer needs to handle this. For tpe, smac and spearmint we stop wrapping.py fixes #25 --- HPOlib/check_before_start.py | 1 + HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py | 1 + HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py | 1 + HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py | 1 + 4 files changed, 4 insertions(+) diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index 0b4fd672..b7475bdd 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -132,6 +132,7 @@ def check_optimizer(optimizer): sys.exit(1) if not os.path.exists(version + ".py"): logger.critical("Sorry I cannot find the script to call your optimizer: %s.py" % version) + sys.exit(1) else: return version diff --git a/HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py b/HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py index 040363f4..241419f5 100644 --- a/HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py +++ b/HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py @@ -78,6 +78,7 @@ def add_default(config): path_to_optimizer = os.path.normpath(path_to_optimizer) if not os.path.exists(path_to_optimizer): logger.critical("Path to optimizer not found: %s" % path_to_optimizer) + sys.exit(1) config.set('SMAC', 'path_to_optimizer', path_to_optimizer) diff --git a/HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py b/HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py index 4d3c27e2..f8695b21 100644 --- a/HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py +++ b/HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py @@ -71,6 +71,7 @@ def add_default(config): path_to_optimizer = os.path.normpath(path_to_optimizer) if not os.path.exists(path_to_optimizer): logger.critical("Path to optimizer not found: %s" % path_to_optimizer) + sys.exit(1) config.set('SPEARMINT', 'path_to_optimizer', path_to_optimizer) diff --git a/HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py b/HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py index ea9f0e29..8a899b4b 100644 --- a/HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py +++ b/HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py @@ -61,6 +61,7 @@ def add_default(config): path_to_optimizer = os.path.normpath(path_to_optimizer) if not os.path.exists(path_to_optimizer): logger.critical("Path to optimizer not found: %s" % path_to_optimizer) + sys.exit(1) config.set('TPE', 'path_to_optimizer', path_to_optimizer) From e681fccd7f4e17d94f8d4826f6bc8d61eedf0aee Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 3 Mar 2014 17:10:43 +0100 Subject: [PATCH 054/139] REFACTOR #16: tpecal.py uses argparse closes #16 --- HPOlib/optimizers/tpe/tpecall.py | 68 +++++++++++++++----------------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/HPOlib/optimizers/tpe/tpecall.py b/HPOlib/optimizers/tpe/tpecall.py index 7eea63a2..8a8b1e41 100644 --- a/HPOlib/optimizers/tpe/tpecall.py +++ b/HPOlib/optimizers/tpe/tpecall.py @@ -17,6 +17,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from argparse import ArgumentParser import cPickle from optparse import OptionParser @@ -24,7 +25,6 @@ from importlib import import_module import logging import os -import sys import hyperopt @@ -59,40 +59,34 @@ def pyll_replace_list_with_dict(search_space, indent = 0): def main(): - # Parse options and arguments - parser = OptionParser() - parser.add_option("-p", "--space", - dest="spaceFile", - help="Where is the space.py located?") - parser.add_option("-a", "--algoExec", - dest="algoExec", - help="Which function to load located?") - parser.add_option("-m", "--maxEvals", - dest="maxEvals", - help="How many evaluations?") - parser.add_option("-s", "--seed", - dest="seed", - default="123", - type=int, - help="Seed for the TPE algorithm") - parser.add_option("-r", "--restore", - dest="restore", - action="store_true", - help="When this flag is set state.pkl is restored in " + + prog = "python statistics.py WhatIsThis WhatIsThis [WhatIsThis ]" + description = "Return some statistical information" + + parser = ArgumentParser(description=description, prog=prog) + + parser.add_argument("-p", "--space", + dest="spaceFile", help="Where is the space.py located?") + parser.add_argument("-a", "--algoExec", + dest="algoExec", help="Which function to load located?") + parser.add_argument("-m", "--maxEvals", + dest="maxEvals", help="How many evaluations?") + parser.add_argument("-s", "--seed", default="1", + dest="seed", type=int, help="Seed for the TPE algorithm") + parser.add_argument("-r", "--restore", action="store_true", + dest="restore", help="When this flag is set state.pkl is restored in " + "the current working directory") - parser.add_option("--random", default=False, - dest="random", - action="store_true", - help="Use a random search") - (options, args) = parser.parse_args() - - if not os.path.exists(options.spaceFile): - logger.critical("Search space not found: %s" % options.spaceFile) + parser.add_argument("--random", default=False, action="store_true", + dest="random", help="Use a random search") + + args, unknown = parser.parse_known_args() + + if not os.path.exists(args.spaceFile): + logger.critical("Search space not found: %s" % args.spaceFile) sys.exit(1) # First remove ".py" - algo, ext = os.path.splitext(os.path.basename(options.algoExec)) - space, ext = os.path.splitext(os.path.basename(options.spaceFile)) + algo, ext = os.path.splitext(os.path.basename(args.algoExec)) + space, ext = os.path.splitext(os.path.basename(args.spaceFile)) # Then load dict searchSpace and out function cv.py import sys @@ -105,16 +99,16 @@ def main(): fn = import_module(algo) fn = fn.doForTPE - if options.random: + if args.random: # We use a random search - tpe_with_seed = partial(hyperopt.tpe.rand.suggest, seed=int(options.seed)) + tpe_with_seed = partial(hyperopt.tpe.rand.suggest, seed=int(args.seed)) logger.info("Using Random Search") else: - tpe_with_seed = partial(hyperopt.tpe.suggest, seed=int(options.seed)) + tpe_with_seed = partial(hyperopt.tpe.suggest, seed=int(args.seed)) # Now run TPE, emulate fmin.fmin() state_filename = "state.pkl" - if options.restore: + if args.restore: # We do not need to care about the state of the trials object since it # is only serialized in a synchronized state, there will never be a save # with a running experiment @@ -124,14 +118,14 @@ def main(): trials = tmp_dict['trials'] print trials.__dict__ else: - domain = hyperopt.Domain(fn, search_space, rseed=int(options.seed)) + domain = hyperopt.Domain(fn, search_space, rseed=int(args.seed)) trials = hyperopt.Trials() fh = open(state_filename, "w") # By this we probably loose the seed; not too critical for a restart cPickle.dump({"trials": trials, "domain": domain}, fh) fh.close() - for i in range(int(options.maxEvals) + 1): + for i in range(int(args.maxEvals) + 1): # in exhaust, the number of evaluations is max_evals - num_done rval = hyperopt.FMinIter(tpe_with_seed, domain, trials, max_evals=i) rval.exhaust() From 73e0710884a77d4771a3e3e6b53098327708221e Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 3 Mar 2014 17:38:32 +0100 Subject: [PATCH 055/139] FIX: wrapping.py waits till optimizer finishes --- HPOlib/wrapping.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 47acfef9..8bddb383 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -344,8 +344,9 @@ def main(): elif args.silent: # Print nothing - proc = subprocess.Popen(cmd, shell=True, stdin=fh, stdout=fh, stderr=fh) + proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=fh, stderr=fh) logger.info('Optimizer runs with PID (+1): %d' % proc.pid) + proc.wait() else: # Print only stderr From 0bc4851f9f823c88d26630f2ba9ea168d3c1e5a1 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 3 Mar 2014 17:52:34 +0100 Subject: [PATCH 056/139] FIX: don't import function in runsolver_wrapper.py Removes legacy code where HPOlib tries to import benchmark function. We don't need this anymore as we run the benchmarks via CLI. --- HPOlib/runsolver_wrapper.py | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 7864ca0f..4edd70b1 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -191,33 +191,20 @@ def parse_command_line(): def get_function_filename(cfg): + fn_path = None + exp_dir = os.path.dirname(os.getcwd()) + + # Check whether function path is absolute if os.path.isabs(cfg.get("HPOLIB", "function")): fn_path = cfg.get("HPOLIB", "function") else: - fn_path = cfg.get("HPOLIB", "function") - fn_path_parent = os.path.join("..", cfg.get("HPOLIB", "function")) - fn_name, ext = os.path.splitext(os.path.basename(fn_path)) - try: - fn = imp.load_source(fn_name, fn_path) - except (ImportError, IOError) as e: - logger.warning("Raised" + e.message + "trying to recover...") - try: - logger.info(os.path.exists(fn_path_parent)) - fn = imp.load_source(fn_name, fn_path_parent) - except (ImportError, IOError): - logger.critical(os.path.join(fn_path_parent)) - logger.critical(("Could not find\n%s\n\tin\n%s\n\tor its parent " - "directory relative to\n%s") - % (fn_name, fn_path, os.getcwd())) - import traceback - - logger.critical(traceback.format_exc()) - sys.exit(1) - fn_filename = os.path.realpath(fn.__file__) - if fn_filename[-3:] == "pyc": - fn_filename = fn_filename[:-1] - - return fn_filename + # We hope to find function in the experiment directory + fn_path = os.path.join(exp_dir, cfg.get("HPOLIB", "function")) + + if not os.path.exists(fn_path): + logger.critical("%s is not absolute nor in %s. Function cannot be found" % + (cfg.get("HPOLIB", "function"), exp_dir)) + return fn_path def make_command(cfg, fold, param_string, run_instance_output): From 896be34b34b47e2d7b3a733882c7cd013738fb86 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 3 Mar 2014 17:56:02 +0100 Subject: [PATCH 057/139] FIX: return, when function cannot be found call sys.exit(1) when runsolver_wrapper cannot find function --- HPOlib/runsolver_wrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 4edd70b1..02050009 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -204,6 +204,8 @@ def get_function_filename(cfg): if not os.path.exists(fn_path): logger.critical("%s is not absolute nor in %s. Function cannot be found" % (cfg.get("HPOLIB", "function"), exp_dir)) + sys.exit(1) + return fn_path From ef598482bef13d7d9a63ffbaf5c84099750680bf Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Tue, 4 Mar 2014 17:03:57 +0100 Subject: [PATCH 058/139] REFACTOR: Move tests into a directory called unittests. By this we can seperate between unittests and integration tests later on. --- HPOlib/cv.py | 2 + tests/test_wrapping.py | 157 ------------------ tests/testsuite.py | 18 +- tests/{ => unittests}/branin_test/__init__.py | 0 tests/{ => unittests}/branin_test/branin.py | 0 tests/{ => unittests}/branin_test/config.cfg | 0 tests/{ => unittests}/branin_test/params123 | 0 .../branin_test/random/space.py | 0 .../branin_test/smac/params.txt | 0 .../branin_test/spearmint/__init__.py | 0 .../branin_test/spearmint/config.pb | 0 .../{ => unittests}/branin_test/tpe/space.py | 0 tests/{ => unittests}/dummy_config.cfg | 5 +- .../run_instance_no_result.txt | 0 tests/{ => unittests}/run_instance_result.txt | 0 tests/{ => unittests}/runsolver_positive.txt | 0 .../runsolver_positive_with_warning.txt | 0 .../runsolver_vsize_exceeded.txt | 0 .../runsolver_wallclock_time_limit.txt | 0 .../{ => unittests}/search_spaces/__init__.py | 0 .../{ => unittests}/search_spaces/autoweka.py | 0 .../{ => unittests}/search_spaces/nips2011.py | 0 tests/{ => unittests}/test_benchmark_util.py | 0 tests/{ => unittests}/test_cv.py | 0 tests/{ => unittests}/test_data_utils.py | 0 tests/{ => unittests}/test_experiment.py | 0 .../{ => unittests}/test_runsolver_wrapper.py | 0 27 files changed, 16 insertions(+), 166 deletions(-) delete mode 100644 tests/test_wrapping.py rename tests/{ => unittests}/branin_test/__init__.py (100%) rename tests/{ => unittests}/branin_test/branin.py (100%) rename tests/{ => unittests}/branin_test/config.cfg (100%) rename tests/{ => unittests}/branin_test/params123 (100%) rename tests/{ => unittests}/branin_test/random/space.py (100%) rename tests/{ => unittests}/branin_test/smac/params.txt (100%) rename tests/{ => unittests}/branin_test/spearmint/__init__.py (100%) rename tests/{ => unittests}/branin_test/spearmint/config.pb (100%) rename tests/{ => unittests}/branin_test/tpe/space.py (100%) rename tests/{ => unittests}/dummy_config.cfg (52%) rename tests/{ => unittests}/run_instance_no_result.txt (100%) rename tests/{ => unittests}/run_instance_result.txt (100%) rename tests/{ => unittests}/runsolver_positive.txt (100%) rename tests/{ => unittests}/runsolver_positive_with_warning.txt (100%) rename tests/{ => unittests}/runsolver_vsize_exceeded.txt (100%) rename tests/{ => unittests}/runsolver_wallclock_time_limit.txt (100%) rename tests/{ => unittests}/search_spaces/__init__.py (100%) rename tests/{ => unittests}/search_spaces/autoweka.py (100%) rename tests/{ => unittests}/search_spaces/nips2011.py (100%) rename tests/{ => unittests}/test_benchmark_util.py (100%) rename tests/{ => unittests}/test_cv.py (100%) rename tests/{ => unittests}/test_data_utils.py (100%) rename tests/{ => unittests}/test_experiment.py (100%) rename tests/{ => unittests}/test_runsolver_wrapper.py (100%) diff --git a/HPOlib/cv.py b/HPOlib/cv.py index 8ffd7dce..231bdd64 100644 --- a/HPOlib/cv.py +++ b/HPOlib/cv.py @@ -58,6 +58,8 @@ def doCV(params, folds=10): cfg = load_experiment_config_file() # Now evaluate $fold times + # TODO This is a bad idea, no one replaces this by the worst possible + # value later on if folds < 1: return np.NaN # Store the results to hand them back to tpe and spearmint diff --git a/tests/test_wrapping.py b/tests/test_wrapping.py deleted file mode 100644 index db64419f..00000000 --- a/tests/test_wrapping.py +++ /dev/null @@ -1,157 +0,0 @@ -## -# wrapping: A program making it easy to use hyperparameter -# optimization software. -# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import shutil -import sys -import unittest -import tempfile -import StringIO - -import HPOlib.wrapping as wrapping -import HPOlib.config_parser.parse as parse - -class WrappingTest(unittest.TestCase): - def setUp(self): - # Change into the test directory - os.chdir(os.path.dirname(os.path.realpath(__file__))) - - # Make sure there is no config file - try: - os.remove("./config.cfg") - except: - pass - - @unittest.skip("Not implemented yet") - def test_calculate_wrapping_overhead(self): - self.fail() - - @unittest.skip("Not implemented yet") - def test_calculate_optimizer_time(self): - self.fail() - - # Test spearmint - @unittest.skip("Not implemented yet") - def test_build_spearmint_call(self): - self.fail() - - @unittest.skip("Not implemented yet") - def test_main_spearmint(self): - self.fail() - - # Test tpe - @unittest.skip("Not implemented yet") - def test_build_tpe_call(self): - self.fail() - - @unittest.skip("Not implemented yet") - def test_main_tpe(self): - self.fail() - - # Test random - @unittest.skip("Not implemented yet") - def test_build_random_call(self): - self.fail() - - @unittest.skip("Not implemented yet") - def test_main_random(self): - self.fail() - - # Test smac - @unittest.skip("Not implemented yet") - def test_build_smac_call(self): - self.fail() - - def test_use_option_parser_no_optimizer(self): - # Test not specifying an optimizer but random other options - sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet'] - self.assertRaises(SystemExit, wrapping.use_arg_parser) - - def test_use_option_parser_the_right_way(self): - sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet', '-o', 'SMAC'] - args, unknown = wrapping.use_arg_parser() - self.assertEqual(args.optimizer, 'SMAC') - self.assertEqual(args.seed, 1) - self.assertEqual(args.title, 'DBNet') - self.assertEqual(len(unknown), 0) - - def test_use_option_parser_with_config(self): - sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet', '-o', 'SMAC', - '--HPOLIB:total_time_limit', '3600'] - args, unknown = wrapping.use_arg_parser() - self.assertEqual(len(unknown), 2) - config = parse.parse_config("dummy_config.cfg", allow_no_value=True) - config_args = wrapping.parse_config_values_from_unknown_arguments( - unknown, config) - self.assertEqual(vars(config_args)['HPOLIB:total_time_limit'], - '3600') - - def test_override_config_with_cli_arguments(self): - unknown = ['--HPOLIB:total_time_limit', '50', - '--HPOLIB:numberofjobs', '2'] - config = parse.parse_config("dummy_config.cfg", allow_no_value=True) - config_args = wrapping.parse_config_values_from_unknown_arguments( - unknown, config) - new_config = wrapping.override_config_with_cli_arguments(config, - config_args) - self.assertEqual(new_config.get('HPOLIB', 'numberofjobs'), '2') - - - def test_save_config_to_file(self): - unknown = ['--HPOLIB:total_time_limit', '50', - '--HPOLIB:numberofjobs', '2'] - config = parse.parse_config("dummy_config.cfg", allow_no_value=True) - string_stream = StringIO.StringIO() - wrapping.save_config_to_file(string_stream, config) - file_content = string_stream.getvalue() - asserted_file_content = "[HPOLIB]\n" \ - "numberofjobs = 1\n" \ - "result_on_terminate = 1\n"\ - "function = 1\n"\ - "algorithm = cv.py\n"\ - "run_instance = runsolver_wrapper.py\n"\ - "numbercv = 1\n"\ - "numberofconcurrentjobs = 1\n"\ - "runsolver_time_limit = 3600\n"\ - "total_time_limit = 3600\n"\ - "memory_limit = 2000\n"\ - "cpu_limit = 14400\n"\ - "max_crash_per_cv = 3\n" - - self.assertEqual(asserted_file_content, file_content) - string_stream.close() - - @unittest.skip("Not implemented yet") - def test_main_smac(self): - os.chdir("./branin_test/") - sys.argv = ['wrapping.py', 'smac'] - ret = wrapping.main() - self.assertEqual(ret, 0, "Return code of SMAC is not 0, please debug") - - # TODO: Test what happens when the runsolver limit is exceeded - # TODO: Clean up the test directory - - # General main test - @unittest.skip("Not implemented yet") - def test_main(self): - self.fail() - - - -if __name__ == "__main__": - unittest.main() \ No newline at end of file diff --git a/tests/testsuite.py b/tests/testsuite.py index a8986923..c820644a 100644 --- a/tests/testsuite.py +++ b/tests/testsuite.py @@ -1,19 +1,21 @@ import unittest -import test_benchmark_util -import test_cv -import test_experiment -import test_data_utils -import test_runsolver_wrapper -import test_wrapping +import unittests.test_benchmark_util as test_benchmark_util +# import unittests.test_cv as test_cv +import unittests.test_data_utils as test_data_utils +import unittests.test_experiment as test_experiment +import unittests.test_gridsearch as test_gridsearch +import unittests.test_runsolver_wrapper as test_runsolver_wrapper +import unittests.test_wrapping as test_wrapping def suite(): _suite = unittest.TestSuite() _suite.addTest(unittest.makeSuite(test_benchmark_util.BenchmarkUtilTest)) - _suite.addTest(unittest.makeSuite(test_cv.CVTest)) - _suite.addTest(unittest.makeSuite(test_experiment.ExperimentTest)) + # _suite.addTest(unittest.makeSuite(test_cv.CVTest)) _suite.addTest(unittest.makeSuite(test_data_utils.DataUtilTest)) + _suite.addTest(unittest.makeSuite(test_experiment.ExperimentTest)) + _suite.addTest(unittest.makeSuite(test_gridsearch.GridSearchTest)) _suite.addTest(unittest.makeSuite(test_runsolver_wrapper.RunsolverWrapperTest)) _suite.addTest(unittest.makeSuite(test_wrapping.WrappingTest)) return _suite diff --git a/tests/branin_test/__init__.py b/tests/unittests/branin_test/__init__.py similarity index 100% rename from tests/branin_test/__init__.py rename to tests/unittests/branin_test/__init__.py diff --git a/tests/branin_test/branin.py b/tests/unittests/branin_test/branin.py similarity index 100% rename from tests/branin_test/branin.py rename to tests/unittests/branin_test/branin.py diff --git a/tests/branin_test/config.cfg b/tests/unittests/branin_test/config.cfg similarity index 100% rename from tests/branin_test/config.cfg rename to tests/unittests/branin_test/config.cfg diff --git a/tests/branin_test/params123 b/tests/unittests/branin_test/params123 similarity index 100% rename from tests/branin_test/params123 rename to tests/unittests/branin_test/params123 diff --git a/tests/branin_test/random/space.py b/tests/unittests/branin_test/random/space.py similarity index 100% rename from tests/branin_test/random/space.py rename to tests/unittests/branin_test/random/space.py diff --git a/tests/branin_test/smac/params.txt b/tests/unittests/branin_test/smac/params.txt similarity index 100% rename from tests/branin_test/smac/params.txt rename to tests/unittests/branin_test/smac/params.txt diff --git a/tests/branin_test/spearmint/__init__.py b/tests/unittests/branin_test/spearmint/__init__.py similarity index 100% rename from tests/branin_test/spearmint/__init__.py rename to tests/unittests/branin_test/spearmint/__init__.py diff --git a/tests/branin_test/spearmint/config.pb b/tests/unittests/branin_test/spearmint/config.pb similarity index 100% rename from tests/branin_test/spearmint/config.pb rename to tests/unittests/branin_test/spearmint/config.pb diff --git a/tests/branin_test/tpe/space.py b/tests/unittests/branin_test/tpe/space.py similarity index 100% rename from tests/branin_test/tpe/space.py rename to tests/unittests/branin_test/tpe/space.py diff --git a/tests/dummy_config.cfg b/tests/unittests/dummy_config.cfg similarity index 52% rename from tests/dummy_config.cfg rename to tests/unittests/dummy_config.cfg index 8a0ba2db..90bd6994 100644 --- a/tests/dummy_config.cfg +++ b/tests/unittests/dummy_config.cfg @@ -1,4 +1,7 @@ [HPOLIB] numberOfJobs = 1 result_on_terminate = 1 -function = 1 \ No newline at end of file +function = 1 + +[GRIDSEARCH] +params = params.pcs \ No newline at end of file diff --git a/tests/run_instance_no_result.txt b/tests/unittests/run_instance_no_result.txt similarity index 100% rename from tests/run_instance_no_result.txt rename to tests/unittests/run_instance_no_result.txt diff --git a/tests/run_instance_result.txt b/tests/unittests/run_instance_result.txt similarity index 100% rename from tests/run_instance_result.txt rename to tests/unittests/run_instance_result.txt diff --git a/tests/runsolver_positive.txt b/tests/unittests/runsolver_positive.txt similarity index 100% rename from tests/runsolver_positive.txt rename to tests/unittests/runsolver_positive.txt diff --git a/tests/runsolver_positive_with_warning.txt b/tests/unittests/runsolver_positive_with_warning.txt similarity index 100% rename from tests/runsolver_positive_with_warning.txt rename to tests/unittests/runsolver_positive_with_warning.txt diff --git a/tests/runsolver_vsize_exceeded.txt b/tests/unittests/runsolver_vsize_exceeded.txt similarity index 100% rename from tests/runsolver_vsize_exceeded.txt rename to tests/unittests/runsolver_vsize_exceeded.txt diff --git a/tests/runsolver_wallclock_time_limit.txt b/tests/unittests/runsolver_wallclock_time_limit.txt similarity index 100% rename from tests/runsolver_wallclock_time_limit.txt rename to tests/unittests/runsolver_wallclock_time_limit.txt diff --git a/tests/search_spaces/__init__.py b/tests/unittests/search_spaces/__init__.py similarity index 100% rename from tests/search_spaces/__init__.py rename to tests/unittests/search_spaces/__init__.py diff --git a/tests/search_spaces/autoweka.py b/tests/unittests/search_spaces/autoweka.py similarity index 100% rename from tests/search_spaces/autoweka.py rename to tests/unittests/search_spaces/autoweka.py diff --git a/tests/search_spaces/nips2011.py b/tests/unittests/search_spaces/nips2011.py similarity index 100% rename from tests/search_spaces/nips2011.py rename to tests/unittests/search_spaces/nips2011.py diff --git a/tests/test_benchmark_util.py b/tests/unittests/test_benchmark_util.py similarity index 100% rename from tests/test_benchmark_util.py rename to tests/unittests/test_benchmark_util.py diff --git a/tests/test_cv.py b/tests/unittests/test_cv.py similarity index 100% rename from tests/test_cv.py rename to tests/unittests/test_cv.py diff --git a/tests/test_data_utils.py b/tests/unittests/test_data_utils.py similarity index 100% rename from tests/test_data_utils.py rename to tests/unittests/test_data_utils.py diff --git a/tests/test_experiment.py b/tests/unittests/test_experiment.py similarity index 100% rename from tests/test_experiment.py rename to tests/unittests/test_experiment.py diff --git a/tests/test_runsolver_wrapper.py b/tests/unittests/test_runsolver_wrapper.py similarity index 100% rename from tests/test_runsolver_wrapper.py rename to tests/unittests/test_runsolver_wrapper.py From 75a04c7bc67e0b813678c0434507c70a7294dac7 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Tue, 4 Mar 2014 17:05:24 +0100 Subject: [PATCH 059/139] ENHANCEMENT: #19 --- HPOlib/config_parser/parse.py | 13 ++- HPOlib/wrapping.py | 55 +++++++++-- tests/unittests/test_wrapping.py | 156 +++++++++++++++++++++++++++++++ 3 files changed, 212 insertions(+), 12 deletions(-) create mode 100644 tests/unittests/test_wrapping.py diff --git a/HPOlib/config_parser/parse.py b/HPOlib/config_parser/parse.py index 157d4011..8e291426 100644 --- a/HPOlib/config_parser/parse.py +++ b/HPOlib/config_parser/parse.py @@ -26,16 +26,19 @@ logger = logging.getLogger("HPOlib.config_parser.parse") -def parse_config(config_fn, allow_no_value=True, optimizer_version=""): - # Reads config_fn +def parse_config(config_file, allow_no_value=True, optimizer_version="", + cli_values=None): + # Reads config_file # Overwrites with default values from generalDefault.cfg # Loads optimizer specific parser, called 'optimizer_version'_parser.py, which can read its own default config - if not os.path.isfile(config_fn): + if not os.path.isfile(config_file): raise Exception('%s is not a valid file\n' % os.path.join( - os.getcwd(), config_fn)) + os.getcwd(), config_file)) config = ConfigParser.SafeConfigParser(allow_no_value=allow_no_value) - config.read(config_fn) + config.read(config_file) + if cli_values is not None: + config.readfp(cli_values) # Load general default configs config_fn_default = \ diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 47acfef9..4d251eea 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -20,6 +20,7 @@ import imp import logging import os +from StringIO import StringIO import subprocess import sys import time @@ -155,6 +156,18 @@ def parse_config_values_from_unknown_arguments(unknown_arguments, config): return parser.parse_args(unknown_arguments) +def config_with_cli_arguments(config, config_overrides): + arg_dict = vars(config_overrides) + for section in config.sections(): + for key in config.options(section): + cli_key = "%s:%s" % (section, key) + if cli_key in arg_dict: + config.set(section, key, arg_dict[cli_key]) + else: + config.remove_option(section, key) + return config + + def override_config_with_cli_arguments(config, config_overrides): arg_dict = vars(config_overrides) for key in arg_dict: @@ -175,18 +188,22 @@ def override_config_with_cli_arguments(config, config_overrides): return config -def save_config_to_file(file_handle, config): +def save_config_to_file(file_handle, config, write_nones=True): if len(config.defaults()) > 0: file_handle.write("[DEFAULT]\n") for key in config.defaults(): - file_handle.write(key + " = " + config.get("DEFAULT", key) + "\n") + if (config.get("DEFAULT", key) is None and write_nones) or \ + config.get("DEFAULT", key) is not None: + file_handle.write(key + " = " + config.get("DEFAULT", key) + "\n") for section in config.sections(): file_handle.write("[" + section + "]\n") for key in config.options(section): - if config.get(section, key) is None or \ - config.get(section, key) != "": - file_handle.write(key + " = " + config.get(section, key) + "\n") + if (config.get(section, key) is None and write_nones) or \ + config.get(section, key) is not None: + if config.get(section, key) is None or \ + config.get(section, key) != "": + file_handle.write("%s = %s\n" % (key, config.get(section, key))) def main(): @@ -212,12 +229,36 @@ def main(): logger.warning("You called -o %s, I am using optimizer defined in %sDefault.cfg" % (optimizer, optimizer_version)) optimizer = os.path.basename(optimizer_version) + """How the configuration is parsed: + 1. The command line is already parsed, we have a list of unknown arguments + 2. call parse_config to find out all allowed keys. Ignore the values. + 3. call parse_config_values_from_unknown_arguments to get parameters for + the optimization software from the command line in a config object + 4. call parse_config again to fill in the missing values + """ config_file = os.path.join(experiment_dir, "config.cfg") + # Is the config file really the right place to get the allowed keys for a + # hyperparameter optimizer? config = parse_config(config_file, allow_no_value=True, optimizer_version=optimizer_version) - # Override config values with command line values + # Parse command line arguments config_overrides = parse_config_values_from_unknown_arguments(unknown, config) - config = override_config_with_cli_arguments(config, config_overrides) + + # Remove every option from the configuration which was not present on the + # command line + config = config_with_cli_arguments(config, config_overrides) + + # Transform to a string so we can pass it to parse_config + fh = StringIO() + save_config_to_file(fh, config, write_nones=False) + fh.seek(0) + + config = parse_config(config_file, allow_no_value=True, + optimizer_version=optimizer_version, + cli_values=fh) + fh.close() + + # config = override_config_with_cli_arguments(config, config_overrides) # Saving the config file is down further at the bottom, as soon as we get # hold of the new optimizer directory wrapping_dir = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/unittests/test_wrapping.py b/tests/unittests/test_wrapping.py new file mode 100644 index 00000000..575ca8a4 --- /dev/null +++ b/tests/unittests/test_wrapping.py @@ -0,0 +1,156 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import shutil +import sys +import unittest +import tempfile +import StringIO + +import HPOlib.wrapping as wrapping +import HPOlib.config_parser.parse as parse + +class WrappingTest(unittest.TestCase): + def setUp(self): + # Change into the test directory + os.chdir(os.path.dirname(os.path.realpath(__file__))) + + # Make sure there is no config file + try: + os.remove("./config.cfg") + except: + pass + + @unittest.skip("Not implemented yet") + def test_calculate_wrapping_overhead(self): + self.fail() + + @unittest.skip("Not implemented yet") + def test_calculate_optimizer_time(self): + self.fail() + + def test_use_option_parser_no_optimizer(self): + # Test not specifying an optimizer but random other options + sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet'] + self.assertRaises(SystemExit, wrapping.use_arg_parser) + + def test_use_option_parser_the_right_way(self): + sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet', '-o', 'SMAC'] + args, unknown = wrapping.use_arg_parser() + self.assertEqual(args.optimizer, 'SMAC') + self.assertEqual(args.seed, 1) + self.assertEqual(args.title, 'DBNet') + self.assertEqual(len(unknown), 0) + + def test_use_option_parser_with_config(self): + sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet', '-o', 'SMAC', + '--HPOLIB:total_time_limit', '3600'] + args, unknown = wrapping.use_arg_parser() + self.assertEqual(len(unknown), 2) + config = parse.parse_config("dummy_config.cfg", allow_no_value=True) + config_args = wrapping.parse_config_values_from_unknown_arguments( + unknown, config) + self.assertEqual(vars(config_args)['HPOLIB:total_time_limit'], + '3600') + + def test_override_config_with_cli_arguments(self): + unknown = ['--HPOLIB:total_time_limit', '50', + '--HPOLIB:numberofjobs', '2'] + config = parse.parse_config("dummy_config.cfg", allow_no_value=True) + config_args = wrapping.parse_config_values_from_unknown_arguments( + unknown, config) + new_config = wrapping.override_config_with_cli_arguments(config, + config_args) + self.assertEqual(new_config.get('HPOLIB', 'numberofjobs'), '2') + + def test_override_config_with_cli_arguments_2(self): + """This tests whether wrapping + .parse_config_values_from_unknown_arguments works as expected. This + test is not sufficient to conclude that we can actually override + optimizer parameters from the command line. We must make sure that + the config parser is invoked before we parse the cli arguments.""" + unknown = ['--HPOLIB:total_time_limit', '50', + '--HPOLIB:numberofjobs', '2', + '--GRIDSEARCH:params', 'test'] + config = parse.parse_config("dummy_config.cfg", allow_no_value=True) + config_args = wrapping.parse_config_values_from_unknown_arguments( + unknown, config) + new_config = wrapping.override_config_with_cli_arguments(config, + config_args) + self.assertEqual(new_config.get('GRIDSEARCH', 'params'), 'test') + + def test_save_config_to_file(self): + unknown = ['--HPOLIB:total_time_limit', '50', + '--HPOLIB:numberofjobs', '2'] + config = parse.parse_config("dummy_config.cfg", allow_no_value=True) + config.set("HPOLIB", "total_time_limit", None) + string_stream = StringIO.StringIO() + wrapping.save_config_to_file(string_stream, config) + file_content = string_stream.getvalue() + asserted_file_content = "[HPOLIB]\n" \ + "numberofjobs = 1\n" \ + "result_on_terminate = 1\n"\ + "function = 1\n"\ + "algorithm = cv.py\n"\ + "run_instance = runsolver_wrapper.py\n"\ + "numbercv = 1\n"\ + "numberofconcurrentjobs = 1\n"\ + "runsolver_time_limit = 3600\n"\ + "total_time_limit = None\n"\ + "memory_limit = 2000\n"\ + "cpu_limit = 14400\n"\ + "max_crash_per_cv = 3\n" \ + "[GRIDSEARCH]\n" \ + "params = params.pcs\n" + + self.assertEqual(asserted_file_content, file_content) + string_stream.close() + + def test_save_config_to_file_ignore_none(self): + config = parse.parse_config("dummy_config.cfg", allow_no_value=True) + config.set("HPOLIB", "total_time_limit", None) + string_stream = StringIO.StringIO() + wrapping.save_config_to_file(string_stream, config, write_nones=False) + file_content = string_stream.getvalue() + asserted_file_content = "[HPOLIB]\n" \ + "numberofjobs = 1\n" \ + "result_on_terminate = 1\n"\ + "function = 1\n"\ + "algorithm = cv.py\n"\ + "run_instance = runsolver_wrapper.py\n"\ + "numbercv = 1\n"\ + "numberofconcurrentjobs = 1\n"\ + "runsolver_time_limit = 3600\n"\ + "memory_limit = 2000\n"\ + "cpu_limit = 14400\n"\ + "max_crash_per_cv = 3\n" \ + "[GRIDSEARCH]\n" \ + "params = params.pcs\n" + + self.assertEqual(asserted_file_content, file_content) + string_stream.close() + + # General main test + @unittest.skip("Not implemented yet") + def test_main(self): + self.fail() + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From c69a0cf82de954b207ff2463f485c53fbf51268f Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Tue, 4 Mar 2014 17:08:12 +0100 Subject: [PATCH 060/139] FIX: Gridsearch is not online yet, do not test it --- tests/testsuite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testsuite.py b/tests/testsuite.py index c820644a..f049029a 100644 --- a/tests/testsuite.py +++ b/tests/testsuite.py @@ -4,7 +4,7 @@ # import unittests.test_cv as test_cv import unittests.test_data_utils as test_data_utils import unittests.test_experiment as test_experiment -import unittests.test_gridsearch as test_gridsearch +#import unittests.test_gridsearch as test_gridsearch import unittests.test_runsolver_wrapper as test_runsolver_wrapper import unittests.test_wrapping as test_wrapping @@ -15,7 +15,7 @@ def suite(): # _suite.addTest(unittest.makeSuite(test_cv.CVTest)) _suite.addTest(unittest.makeSuite(test_data_utils.DataUtilTest)) _suite.addTest(unittest.makeSuite(test_experiment.ExperimentTest)) - _suite.addTest(unittest.makeSuite(test_gridsearch.GridSearchTest)) + #_suite.addTest(unittest.makeSuite(test_gridsearch.GridSearchTest)) _suite.addTest(unittest.makeSuite(test_runsolver_wrapper.RunsolverWrapperTest)) _suite.addTest(unittest.makeSuite(test_wrapping.WrappingTest)) return _suite From 04a6411e4d91f38930519d5c419ca000c590a647 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 4 Mar 2014 17:12:05 +0100 Subject: [PATCH 061/139] ENHANCEMENT: #18, #4 setup.py loads benchmarks and optimizers when using python setup.py install * branin and hartmann6 benchmarks are copied to cwd * optimizer folder is copied to cwd * optimizer sources are downloaded from automl.org --- setup.py | 251 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 174 insertions(+), 77 deletions(-) diff --git a/setup.py b/setup.py index c037fc32..e0c634eb 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ import hashlib from setuptools import setup -from setuptools.command.test import test as test_command from setuptools.command.install import install as install_command import shutil import urllib @@ -17,15 +16,16 @@ package_dir = {'HPOlib': 'HPOlib', 'HPOlib.config_parser': 'HPOlib/config_parser', - 'HPOlib.benchmarks': 'HPOlib/benchmarks', - 'HPOlib.optimizers': 'HPOlib/optimizers', - 'HPOlib.benchmarks.branin': 'HPOlib/benchmarks/branin', - 'HPOlib.benchmarks.har6': 'HPOlib/benchmarks/har6', + #'HPOlib.benchmarks': 'HPOlib/benchmarks', + #'HPOlib.optimizers': 'HPOlib/optimizers', + #'HPOlib.benchmarks.branin': 'HPOlib/benchmarks/branin', + #'HPOlib.benchmarks.har6': 'HPOlib/benchmarks/har6', 'HPOlib.Plotting': 'HPOlib/Plotting'} -package_data = {'HPOlib': ['*/params.txt', '*/space.py', '*/config.pb'], - 'HPOlib.benchmarks.branin': ['*.cfg', '*/params.txt', '*/space.py', '*/config.pb'], - 'HPOlib.benchmarks.har6': ['*.cfg', '*/params.txt', '*/space.py', '*/config.pb'], - 'HPOlib.config_parser': ['*.cfg']} +package_data = {'HPOlib.config_parser': ['*.cfg']} + # 'HPOlib': ['*/params.txt', '*/space.py', '*/config.pb'], + # 'HPOlib.benchmarks.branin': ['*.cfg', '*/params.txt', '*/space.py', '*/config.pb'], + # 'HPOlib.benchmarks.har6': ['*.cfg', '*/params.txt', '*/space.py', '*/config.pb']} + data_files = [] scripts = ['scripts/HPOlib-run', 'scripts/HPOlib-plot', 'runsolver/src/runsolver'] @@ -37,21 +37,68 @@ def read(fname): def get_find_packages(): packages = ['HPOlib', 'HPOlib.config_parser', - 'HPOlib.benchmarks', - 'HPOlib.optimizers', - 'HPOlib.benchmarks.branin', - 'HPOlib.benchmarks.har6', + #'HPOlib.benchmarks', + #'HPOlib.optimizers', + #'HPOlib.benchmarks.branin', + #'HPOlib.benchmarks.har6', 'HPOlib.Plotting'] return packages -class InstallRunsolver(install_command): - """Is supposed to install the runsolver later on""" - runsolver_needs_to_be_installed = False - download_url = 'http://www.cril.univ-artois.fr/~roussel/runsolver/runsolver-3.3.2.tar.bz2' - md5_should = '62d139a6d2b1b376eb9ed59ccb5d6726' - save_as = os.path.join(os.getcwd(), "runsolver-3.3.2.tar.bz2") - extract_as = os.getcwd() +def download_source(download_url, md5, save_as): + try: + urllib.urlretrieve(download_url, filename=save_as) + except Exception, e: + print("Error downloading %s: %s" % (download_url, e)) + return False + md5_new = hashlib.md5(open(save_as).read()).hexdigest() + if md5_new != md5: + print "md5 checksum has changed: %s to %s" % (md5, md5_new) + return False + return True + + +def extract_source(fn_name, extract_as): + if tarfile.is_tarfile(fn_name): + try: + tfile = tarfile.open(fn_name) + tfile.extractall(extract_as) + except Exception, e: + print("Error occurred during extraction: %s" % e) + return False + return True + return False + + +def copy_folder(new_dir, old_dir): + try: + shutil.copytree(old_dir, new_dir, symlinks=True, + ignore=shutil.ignore_patterns('*.pyc', '*_src*')) + except shutil.Error as e: + sys.stderr.write("[shutil.ERROR] Could not copy folder from %s to %s\n" % (old_dir, new_dir)) + return False + except Exception, e: + sys.stderr.write("[ERROR] Could not copy folder from %s to %s\n" % (old_dir, new_dir)) + sys.stderr.write("%s\n" % e.message) + return False + return True + + +def make_dir(new_dir): + try: + if not os.path.exists(new_dir): + os.mkdir(new_dir) + else: + sys.stdout.write("[INFO] %s is already a directory, we use it for our installation\n" % + new_dir) + except Exception, e: + sys.stdout.write("[ERROR] Something (%s) went wrong, please create %s and try again\n" % + (e.message, new_dir)) + + +class AdditionalInstall(install_command): + """Is supposed to install the runsolver, download optimizers, + and copy benchmarks""" def _check_for_runsolver(self): name = 'runsolver' @@ -67,33 +114,10 @@ def _check_for_runsolver(self): return p return False - def _download_runsolver(self): - try: - urllib.urlretrieve(self.download_url, filename=self.save_as) - except Exception, e: - print("Error downloading %s: %s" % (self.download_url, e)) - return False - md5 = hashlib.md5(open(self.save_as).read()).hexdigest() - if md5 != self.md5_should: - print "md5 checksum has changed: %s to %s" % (self.md5_should, md5) - return False - return True - - def _extract_runsolver(self): - if tarfile.is_tarfile(self.save_as): - try: - tfile = tarfile.open(self.save_as) - tfile.extractall(self.extract_as) - except Exception, e: - print("Error occurred during extraction: %s" % e) - return False - return True - return False - - def _build(self): + def _build(self, build_dir): print("Building runsolver") cur_pwd = os.getcwd() - os.chdir(os.path.join(self.extract_as, "runsolver", "src")) + os.chdir(build_dir) try: subprocess.check_call("make") except subprocess.CalledProcessError, e: @@ -103,8 +127,32 @@ def _build(self): os.chdir(cur_pwd) return True - def run(self): + def _copy_and_download_optimizer(self, new_optimizer_dir, old_optimizer_dir, optimizer_name, optimizer_tar_name, + url, md5): + load, extract, copy = (False, False, False) + copy = copy_folder(old_dir=os.path.join(old_optimizer_dir, optimizer_name), + new_dir=os.path.join(new_optimizer_dir, optimizer_name)) + + load = download_source(download_url=url, md5=md5, + save_as=os.path.join(new_optimizer_dir, optimizer_tar_name)) + if load: + extract = extract_source(os.path.join(new_optimizer_dir, optimizer_tar_name), + extract_as=os.path.join(new_optimizer_dir, optimizer_name)) + os.remove(os.path.join(new_optimizer_dir, optimizer_tar_name)) + + + + if load and extract and copy: + return True + else: + return False + + def run(self): + # RUNSOLVER STUFF + here_we_are = os.getcwd() + runsolver_tar_name = "runsolver-3.3.2.tar.bz2" + runsolver_name = "runsolver-3.3.2" if sys.version_info < (2, 7, 0) or sys.version_info >= (2, 8, 0): sys.stderr.write("HPOlib requires Python 2.7.0\n") sys.exit(-1) @@ -112,26 +160,27 @@ def run(self): downloaded, extracted, built = (False, False, False) runsolver_needs_to_be_installed = False - print "Check whether you already have installed runsolver" runsolver_path = self._check_for_runsolver() if not runsolver_path: - print "'runsolver' not found, so we try to install it" + sys.stdout.write("[INFO] 'runsolver' not found, so we try to install it") runsolver_needs_to_be_installed = True - download_url = 'http://www.cril.univ-artois.fr/~roussel/runsolver/runsolver-3.3.2.tar.bz2' - save_as = os.path.join(os.getcwd(), "runsolver-3.3.2.tar.bz2") - extract_as = os.getcwd() if runsolver_needs_to_be_installed: - print("Downloading runsolver from %s, saving to %s" % (download_url, save_as)) - downloaded = self._download_runsolver() + sys.stdout.write("Downloading runsolver from %s%s, saving to %s/%s" % + ('http://www.cril.univ-artois.fr/~roussel/runsolver/', + runsolver_tar_name, os.getcwd(), runsolver_tar_name)) + downloaded = download_source(download_url='http://www.cril.univ-artois.fr/~roussel/runsolver/%s' % + runsolver_tar_name, + md5="62d139a6d2b1b376eb9ed59ccb5d6726", + save_as=os.path.join(here_we_are, runsolver_tar_name)) if runsolver_needs_to_be_installed and downloaded: - print("Extracting runsolver to %s" % extract_as) - extracted = self._extract_runsolver() + print("Extracting runsolver to %s" % os.path.join(here_we_are, runsolver_name)) + extracted = extract_source(fn_name=os.path.join(here_we_are, runsolver_tar_name), extract_as=here_we_are) if runsolver_needs_to_be_installed and extracted: print("Building runsolver") - built = self._build() + built = self._build(build_dir=os.path.join(here_we_are, "runsolver", "src")) if not built: print "Try a second time and replace '/usr/include/asm/unistd.h' with '/usr/include/unistd.h'" call = "sed -i 's/\/usr\/include\/asm\/unistd/\/usr\/include\/unistd/g' runsolver/src/Makefile" @@ -139,13 +188,72 @@ def run(self): subprocess.check_call(call, shell=True) except subprocess.CalledProcessError, e: print "Replacing did not work: %s" % e - built = self._build() + built = self._build(build_dir=os.path.join(here_we_are, "runsolver", "src")) + + # COPY BENCHMARKS TO ROOT FOLDER + new_benchmark_dir = os.path.join(here_we_are, "benchmarks") + old_benchmark_dir = os.path.join(os.path.dirname(HPOlib.__file__), "benchmarks") + + make_dir(new_dir=new_benchmark_dir) + branin = copy_folder(new_dir=os.path.join(new_benchmark_dir, "branin"), + old_dir=os.path.join(old_benchmark_dir, "branin")) + har6 = copy_folder(new_dir=os.path.join(new_benchmark_dir, "har6"), + old_dir=os.path.join(old_benchmark_dir, "har6")) + + # COPY/DOWNLOAD OPTIMIZER TO ROOT FOLDER + new_optimizer_dir = os.path.join(here_we_are, "optimizers") + old_optimizer_dir = os.path.join(os.path.dirname(HPOlib.__file__), "optimizers") + make_dir(new_dir=new_optimizer_dir) + tpe, smac, spearmint = (False, False, False) + tpe = self._copy_and_download_optimizer(new_optimizer_dir=new_optimizer_dir, + old_optimizer_dir=old_optimizer_dir, + optimizer_name='tpe', + optimizer_tar_name="hyperopt_august2013_mod_src.tar.gz", + url="http://www.automl.org/hyperopt_august2013_mod_src.tar.gz", + md5='15ce1adf9d32bb7f71dcfb9a847c55c7') + # If one optimizer fails, others are likely to fail as well + if tpe: + smac = self._copy_and_download_optimizer(new_optimizer_dir=new_optimizer_dir, + old_optimizer_dir=old_optimizer_dir, + optimizer_name='smac', + optimizer_tar_name="smac_2_06_01-dev_src.tar.gz", + url="http://www.automl.org/smac_2_06_01-dev_src.tar.gz", + md5='30ab1d09696de47efac77ed163772c0a') + spearmint = self._copy_and_download_optimizer(new_optimizer_dir=new_optimizer_dir, + old_optimizer_dir=old_optimizer_dir, + optimizer_name='spearmint', + optimizer_tar_name="spearmint_april2013_mod_src.tar.gz", + url="http://www.automl.org/spearmint_april2013_mod_src.tar.gz", + md5='aa3eb7961637f9bf2a8cb9b14c4b8791') + + # Do whatever you want to do install_command.run(self) # Give detailed output to user + if not har6 or not branin: + sys.stderr.write("[ERROR] Branin/Har6 benchmark could not be copied, please do the following:\n" + + "mkdir benchmarks\n" + + "cp HPOlib/benchmarks/* benchmarks/ -r\n") + if not tpe or not smac or not spearmint: + sys.stderr.write("[ERROR] Something went wrong while copying and downloading optimizers." + + "Please do the following to be ready to start optimizing:\n\n" + + "rm optimizers\n" + "mkdir optimizers\n" + + "cp HPOlib/optimizers/tpe optimizers/ -r\n" + + "cp HPOlib/optimizers/smac optimizers/ -r\n" + + "cp HPOlib/optimizers/spearmint optimizers/ -r\n" + "cd optimizers\n" + + "wget http://www.automl.org/hyperopt_august2013_mod_src.tar.gz \n" + + "wget http://www.automl.org/smac_2_06_01-dev_src.tar.gz \n" + + "wget http://www.automl.org/spearmint_april2013_mod_src.tar.gz \n" + "tar -xf hyperopt_august2013_mod_src.tar.gz \n" + "mv hyperopt_august2013_mod_src tpe/ \n" + + "tar -xf smac_2_06_01-dev_src.tar.gz \n" + "mv smac_2_06_01-dev_src.tar.gz smac/ \n" + + "tar -xf spearmint_april2013_mod_src.tar.gz \n" + + "mv spearmint_april2013_mod_src.tar.gz spearmint/ \n\n" + + "Thank You!\n") if runsolver_needs_to_be_installed and not built: - print "[ERROR] Please install runsolver on your own! You can download it from:\n%s" % download_url + print "[ERROR] Please install runsolver on your own! You can download it from:\n%s%s" % \ + ('http://www.cril.univ-artois.fr/~roussel/runsolver/%s', runsolver_tar_name) print "[ERROR] Be sure 'runsolver' can be found during runtime in your $PATH variable!" if not runsolver_needs_to_be_installed and not built: print "[INFO] 'runsolver' has been found on this system and was copied from: %s" % runsolver_path @@ -153,17 +261,6 @@ def run(self): print "'runsolver' has been downloaded and installed" -class PyTest(test_command): - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - print("Hello User,\n" - "so far we have not included any tests. Sorry") - setup( name='HPOlib', version=HPOlib.__version__, @@ -172,13 +269,13 @@ def run(self): platforms=['Linux'], author=HPOlib.__authors__, test_suite="tests.testsuite.suite", - install_requires=['argparse==1.2.1', - 'matplotlib==1.3.1', - 'networkx==1.8.1', - 'numpy==1.8.0', - 'protobuf==2.5.0', - 'scipy==0.13.2', - 'pymongo==2.6.3', + install_requires=['argparse>=1.2.1', + 'matplotlib>=1.3.1', + 'networkx>=1.8.1', + 'numpy>=1.8.0', + 'protobuf>=2.5.0', + 'scipy>=0.13.2', + 'pymongo>=2.6.3', ], author_email='eggenspk@informatik.uni-freiburg.de', description=desc, @@ -190,7 +287,7 @@ def run(self): data_files=data_files, scripts=scripts, cmdclass={ - 'install': InstallRunsolver, + 'install': AdditionalInstall, }, classifiers=[ 'Programming Language :: Python :: 2.7', From ad9a26fd33322eecee951d4bfe47201b8033caa1 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 4 Mar 2014 17:22:47 +0100 Subject: [PATCH 062/139] REFACTOR: move optimizer and toy benchmark directory Moves HPOlib/benchmarks to ./benchmarks as the HPOlib does not depend on this Same holds for HPOlib/optimizers --- {HPOlib/benchmarks => benchmarks}/__init__.py | 0 .../branin/__init__.py | 0 .../branin/branin.py | 0 .../branin/config.cfg | 0 .../branin/hyperopt_august2013_mod/space.py | 0 .../branin/random_hyperopt_august2013_mod | 0 .../branin/smac_2_06_01-dev/params.txt | 0 .../branin/spearmint_april2013_mod/config.pb | 0 .../har6/__init__.py | 0 .../benchmarks => benchmarks}/har6/config.cfg | 0 .../benchmarks => benchmarks}/har6/har6.py | 0 .../har6/hyperopt_august2013_mod/space.py | 0 {HPOlib/benchmarks => benchmarks}/har6/smac | 0 .../har6/smac_2_06_01-dev/params.txt | 0 .../benchmarks => benchmarks}/har6/spearmint | 0 .../har6/spearmint_april2013_mod/config.pb | 0 {HPOlib/benchmarks => benchmarks}/har6/tpe | 0 {HPOlib/optimizers => optimizers}/__init__.py | 0 .../smac/smac_2_06_01-dev.py | 0 .../smac/smac_2_06_01-devDefault.cfg | 0 .../smac/smac_2_06_01-dev_parser.py | 0 .../spearmint/spearmint_april2013_mod.py | 0 .../spearmint_april2013_modDefault.cfg | 0 .../spearmint_april2013_mod_parser.py | 0 .../tpe/hyperopt_august2013_mod.py | 0 .../tpe/hyperopt_august2013_modDefault.cfg | 0 .../tpe/hyperopt_august2013_mod_parser.py | 0 .../tpe/random_hyperopt_august2013_mod.py | 0 .../random_hyperopt_august2013_modDefault.cfg | 0 .../random_hyperopt_august2013_mod_parser.py | 0 .../optimizers => optimizers}/tpe/tpecall.py | 0 setup.py | 45 +++++-------------- 32 files changed, 11 insertions(+), 34 deletions(-) rename {HPOlib/benchmarks => benchmarks}/__init__.py (100%) rename {HPOlib/benchmarks => benchmarks}/branin/__init__.py (100%) rename {HPOlib/benchmarks => benchmarks}/branin/branin.py (100%) rename {HPOlib/benchmarks => benchmarks}/branin/config.cfg (100%) rename {HPOlib/benchmarks => benchmarks}/branin/hyperopt_august2013_mod/space.py (100%) rename {HPOlib/benchmarks => benchmarks}/branin/random_hyperopt_august2013_mod (100%) rename {HPOlib/benchmarks => benchmarks}/branin/smac_2_06_01-dev/params.txt (100%) rename {HPOlib/benchmarks => benchmarks}/branin/spearmint_april2013_mod/config.pb (100%) rename {HPOlib/benchmarks => benchmarks}/har6/__init__.py (100%) rename {HPOlib/benchmarks => benchmarks}/har6/config.cfg (100%) rename {HPOlib/benchmarks => benchmarks}/har6/har6.py (100%) rename {HPOlib/benchmarks => benchmarks}/har6/hyperopt_august2013_mod/space.py (100%) rename {HPOlib/benchmarks => benchmarks}/har6/smac (100%) rename {HPOlib/benchmarks => benchmarks}/har6/smac_2_06_01-dev/params.txt (100%) rename {HPOlib/benchmarks => benchmarks}/har6/spearmint (100%) rename {HPOlib/benchmarks => benchmarks}/har6/spearmint_april2013_mod/config.pb (100%) rename {HPOlib/benchmarks => benchmarks}/har6/tpe (100%) rename {HPOlib/optimizers => optimizers}/__init__.py (100%) rename {HPOlib/optimizers => optimizers}/smac/smac_2_06_01-dev.py (100%) rename {HPOlib/optimizers => optimizers}/smac/smac_2_06_01-devDefault.cfg (100%) rename {HPOlib/optimizers => optimizers}/smac/smac_2_06_01-dev_parser.py (100%) rename {HPOlib/optimizers => optimizers}/spearmint/spearmint_april2013_mod.py (100%) rename {HPOlib/optimizers => optimizers}/spearmint/spearmint_april2013_modDefault.cfg (100%) rename {HPOlib/optimizers => optimizers}/spearmint/spearmint_april2013_mod_parser.py (100%) rename {HPOlib/optimizers => optimizers}/tpe/hyperopt_august2013_mod.py (100%) rename {HPOlib/optimizers => optimizers}/tpe/hyperopt_august2013_modDefault.cfg (100%) rename {HPOlib/optimizers => optimizers}/tpe/hyperopt_august2013_mod_parser.py (100%) rename {HPOlib/optimizers => optimizers}/tpe/random_hyperopt_august2013_mod.py (100%) rename {HPOlib/optimizers => optimizers}/tpe/random_hyperopt_august2013_modDefault.cfg (100%) rename {HPOlib/optimizers => optimizers}/tpe/random_hyperopt_august2013_mod_parser.py (100%) rename {HPOlib/optimizers => optimizers}/tpe/tpecall.py (100%) diff --git a/HPOlib/benchmarks/__init__.py b/benchmarks/__init__.py similarity index 100% rename from HPOlib/benchmarks/__init__.py rename to benchmarks/__init__.py diff --git a/HPOlib/benchmarks/branin/__init__.py b/benchmarks/branin/__init__.py similarity index 100% rename from HPOlib/benchmarks/branin/__init__.py rename to benchmarks/branin/__init__.py diff --git a/HPOlib/benchmarks/branin/branin.py b/benchmarks/branin/branin.py similarity index 100% rename from HPOlib/benchmarks/branin/branin.py rename to benchmarks/branin/branin.py diff --git a/HPOlib/benchmarks/branin/config.cfg b/benchmarks/branin/config.cfg similarity index 100% rename from HPOlib/benchmarks/branin/config.cfg rename to benchmarks/branin/config.cfg diff --git a/HPOlib/benchmarks/branin/hyperopt_august2013_mod/space.py b/benchmarks/branin/hyperopt_august2013_mod/space.py similarity index 100% rename from HPOlib/benchmarks/branin/hyperopt_august2013_mod/space.py rename to benchmarks/branin/hyperopt_august2013_mod/space.py diff --git a/HPOlib/benchmarks/branin/random_hyperopt_august2013_mod b/benchmarks/branin/random_hyperopt_august2013_mod similarity index 100% rename from HPOlib/benchmarks/branin/random_hyperopt_august2013_mod rename to benchmarks/branin/random_hyperopt_august2013_mod diff --git a/HPOlib/benchmarks/branin/smac_2_06_01-dev/params.txt b/benchmarks/branin/smac_2_06_01-dev/params.txt similarity index 100% rename from HPOlib/benchmarks/branin/smac_2_06_01-dev/params.txt rename to benchmarks/branin/smac_2_06_01-dev/params.txt diff --git a/HPOlib/benchmarks/branin/spearmint_april2013_mod/config.pb b/benchmarks/branin/spearmint_april2013_mod/config.pb similarity index 100% rename from HPOlib/benchmarks/branin/spearmint_april2013_mod/config.pb rename to benchmarks/branin/spearmint_april2013_mod/config.pb diff --git a/HPOlib/benchmarks/har6/__init__.py b/benchmarks/har6/__init__.py similarity index 100% rename from HPOlib/benchmarks/har6/__init__.py rename to benchmarks/har6/__init__.py diff --git a/HPOlib/benchmarks/har6/config.cfg b/benchmarks/har6/config.cfg similarity index 100% rename from HPOlib/benchmarks/har6/config.cfg rename to benchmarks/har6/config.cfg diff --git a/HPOlib/benchmarks/har6/har6.py b/benchmarks/har6/har6.py similarity index 100% rename from HPOlib/benchmarks/har6/har6.py rename to benchmarks/har6/har6.py diff --git a/HPOlib/benchmarks/har6/hyperopt_august2013_mod/space.py b/benchmarks/har6/hyperopt_august2013_mod/space.py similarity index 100% rename from HPOlib/benchmarks/har6/hyperopt_august2013_mod/space.py rename to benchmarks/har6/hyperopt_august2013_mod/space.py diff --git a/HPOlib/benchmarks/har6/smac b/benchmarks/har6/smac similarity index 100% rename from HPOlib/benchmarks/har6/smac rename to benchmarks/har6/smac diff --git a/HPOlib/benchmarks/har6/smac_2_06_01-dev/params.txt b/benchmarks/har6/smac_2_06_01-dev/params.txt similarity index 100% rename from HPOlib/benchmarks/har6/smac_2_06_01-dev/params.txt rename to benchmarks/har6/smac_2_06_01-dev/params.txt diff --git a/HPOlib/benchmarks/har6/spearmint b/benchmarks/har6/spearmint similarity index 100% rename from HPOlib/benchmarks/har6/spearmint rename to benchmarks/har6/spearmint diff --git a/HPOlib/benchmarks/har6/spearmint_april2013_mod/config.pb b/benchmarks/har6/spearmint_april2013_mod/config.pb similarity index 100% rename from HPOlib/benchmarks/har6/spearmint_april2013_mod/config.pb rename to benchmarks/har6/spearmint_april2013_mod/config.pb diff --git a/HPOlib/benchmarks/har6/tpe b/benchmarks/har6/tpe similarity index 100% rename from HPOlib/benchmarks/har6/tpe rename to benchmarks/har6/tpe diff --git a/HPOlib/optimizers/__init__.py b/optimizers/__init__.py similarity index 100% rename from HPOlib/optimizers/__init__.py rename to optimizers/__init__.py diff --git a/HPOlib/optimizers/smac/smac_2_06_01-dev.py b/optimizers/smac/smac_2_06_01-dev.py similarity index 100% rename from HPOlib/optimizers/smac/smac_2_06_01-dev.py rename to optimizers/smac/smac_2_06_01-dev.py diff --git a/HPOlib/optimizers/smac/smac_2_06_01-devDefault.cfg b/optimizers/smac/smac_2_06_01-devDefault.cfg similarity index 100% rename from HPOlib/optimizers/smac/smac_2_06_01-devDefault.cfg rename to optimizers/smac/smac_2_06_01-devDefault.cfg diff --git a/HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py b/optimizers/smac/smac_2_06_01-dev_parser.py similarity index 100% rename from HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py rename to optimizers/smac/smac_2_06_01-dev_parser.py diff --git a/HPOlib/optimizers/spearmint/spearmint_april2013_mod.py b/optimizers/spearmint/spearmint_april2013_mod.py similarity index 100% rename from HPOlib/optimizers/spearmint/spearmint_april2013_mod.py rename to optimizers/spearmint/spearmint_april2013_mod.py diff --git a/HPOlib/optimizers/spearmint/spearmint_april2013_modDefault.cfg b/optimizers/spearmint/spearmint_april2013_modDefault.cfg similarity index 100% rename from HPOlib/optimizers/spearmint/spearmint_april2013_modDefault.cfg rename to optimizers/spearmint/spearmint_april2013_modDefault.cfg diff --git a/HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py b/optimizers/spearmint/spearmint_april2013_mod_parser.py similarity index 100% rename from HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py rename to optimizers/spearmint/spearmint_april2013_mod_parser.py diff --git a/HPOlib/optimizers/tpe/hyperopt_august2013_mod.py b/optimizers/tpe/hyperopt_august2013_mod.py similarity index 100% rename from HPOlib/optimizers/tpe/hyperopt_august2013_mod.py rename to optimizers/tpe/hyperopt_august2013_mod.py diff --git a/HPOlib/optimizers/tpe/hyperopt_august2013_modDefault.cfg b/optimizers/tpe/hyperopt_august2013_modDefault.cfg similarity index 100% rename from HPOlib/optimizers/tpe/hyperopt_august2013_modDefault.cfg rename to optimizers/tpe/hyperopt_august2013_modDefault.cfg diff --git a/HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py b/optimizers/tpe/hyperopt_august2013_mod_parser.py similarity index 100% rename from HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py rename to optimizers/tpe/hyperopt_august2013_mod_parser.py diff --git a/HPOlib/optimizers/tpe/random_hyperopt_august2013_mod.py b/optimizers/tpe/random_hyperopt_august2013_mod.py similarity index 100% rename from HPOlib/optimizers/tpe/random_hyperopt_august2013_mod.py rename to optimizers/tpe/random_hyperopt_august2013_mod.py diff --git a/HPOlib/optimizers/tpe/random_hyperopt_august2013_modDefault.cfg b/optimizers/tpe/random_hyperopt_august2013_modDefault.cfg similarity index 100% rename from HPOlib/optimizers/tpe/random_hyperopt_august2013_modDefault.cfg rename to optimizers/tpe/random_hyperopt_august2013_modDefault.cfg diff --git a/HPOlib/optimizers/tpe/random_hyperopt_august2013_mod_parser.py b/optimizers/tpe/random_hyperopt_august2013_mod_parser.py similarity index 100% rename from HPOlib/optimizers/tpe/random_hyperopt_august2013_mod_parser.py rename to optimizers/tpe/random_hyperopt_august2013_mod_parser.py diff --git a/HPOlib/optimizers/tpe/tpecall.py b/optimizers/tpe/tpecall.py similarity index 100% rename from HPOlib/optimizers/tpe/tpecall.py rename to optimizers/tpe/tpecall.py diff --git a/setup.py b/setup.py index e0c634eb..ce91a156 100644 --- a/setup.py +++ b/setup.py @@ -127,23 +127,19 @@ def _build(self, build_dir): os.chdir(cur_pwd) return True - def _copy_and_download_optimizer(self, new_optimizer_dir, old_optimizer_dir, optimizer_name, optimizer_tar_name, + def _copy_and_download_optimizer(self, optimizer_dir, optimizer_name, optimizer_tar_name, url, md5): - load, extract, copy = (False, False, False) - copy = copy_folder(old_dir=os.path.join(old_optimizer_dir, optimizer_name), - new_dir=os.path.join(new_optimizer_dir, optimizer_name)) + load, extract = (False, False) load = download_source(download_url=url, md5=md5, - save_as=os.path.join(new_optimizer_dir, optimizer_tar_name)) + save_as=os.path.join(optimizer_dir, optimizer_tar_name)) if load: - extract = extract_source(os.path.join(new_optimizer_dir, optimizer_tar_name), - extract_as=os.path.join(new_optimizer_dir, optimizer_name)) - os.remove(os.path.join(new_optimizer_dir, optimizer_tar_name)) + extract = extract_source(os.path.join(optimizer_dir, optimizer_tar_name), + extract_as=os.path.join(optimizer_dir, optimizer_name)) + os.remove(os.path.join(optimizer_dir, optimizer_tar_name)) - - - if load and extract and copy: + if load and extract: return True else: return False @@ -190,38 +186,23 @@ def run(self): print "Replacing did not work: %s" % e built = self._build(build_dir=os.path.join(here_we_are, "runsolver", "src")) - # COPY BENCHMARKS TO ROOT FOLDER - new_benchmark_dir = os.path.join(here_we_are, "benchmarks") - old_benchmark_dir = os.path.join(os.path.dirname(HPOlib.__file__), "benchmarks") - - make_dir(new_dir=new_benchmark_dir) - branin = copy_folder(new_dir=os.path.join(new_benchmark_dir, "branin"), - old_dir=os.path.join(old_benchmark_dir, "branin")) - har6 = copy_folder(new_dir=os.path.join(new_benchmark_dir, "har6"), - old_dir=os.path.join(old_benchmark_dir, "har6")) - # COPY/DOWNLOAD OPTIMIZER TO ROOT FOLDER - new_optimizer_dir = os.path.join(here_we_are, "optimizers") - old_optimizer_dir = os.path.join(os.path.dirname(HPOlib.__file__), "optimizers") - make_dir(new_dir=new_optimizer_dir) + optimizer_dir = os.path.join(here_we_are, "optimizers") tpe, smac, spearmint = (False, False, False) - tpe = self._copy_and_download_optimizer(new_optimizer_dir=new_optimizer_dir, - old_optimizer_dir=old_optimizer_dir, + tpe = self._copy_and_download_optimizer(optimizer_dir=optimizer_dir, optimizer_name='tpe', optimizer_tar_name="hyperopt_august2013_mod_src.tar.gz", url="http://www.automl.org/hyperopt_august2013_mod_src.tar.gz", md5='15ce1adf9d32bb7f71dcfb9a847c55c7') # If one optimizer fails, others are likely to fail as well if tpe: - smac = self._copy_and_download_optimizer(new_optimizer_dir=new_optimizer_dir, - old_optimizer_dir=old_optimizer_dir, + smac = self._copy_and_download_optimizer(optimizer_dir=optimizer_dir, optimizer_name='smac', optimizer_tar_name="smac_2_06_01-dev_src.tar.gz", url="http://www.automl.org/smac_2_06_01-dev_src.tar.gz", md5='30ab1d09696de47efac77ed163772c0a') - spearmint = self._copy_and_download_optimizer(new_optimizer_dir=new_optimizer_dir, - old_optimizer_dir=old_optimizer_dir, + spearmint = self._copy_and_download_optimizer(optimizer_dir=optimizer_dir, optimizer_name='spearmint', optimizer_tar_name="spearmint_april2013_mod_src.tar.gz", url="http://www.automl.org/spearmint_april2013_mod_src.tar.gz", @@ -231,10 +212,6 @@ def run(self): install_command.run(self) # Give detailed output to user - if not har6 or not branin: - sys.stderr.write("[ERROR] Branin/Har6 benchmark could not be copied, please do the following:\n" + - "mkdir benchmarks\n" + - "cp HPOlib/benchmarks/* benchmarks/ -r\n") if not tpe or not smac or not spearmint: sys.stderr.write("[ERROR] Something went wrong while copying and downloading optimizers." + "Please do the following to be ready to start optimizing:\n\n" + From 056006816ebad9850195bccd7d2284bb3c52854a Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Tue, 4 Mar 2014 19:09:40 +0100 Subject: [PATCH 063/139] REFACTOR: issue #26 The config parse code is now much easier and less long. --- HPOlib/check_before_start.py | 4 +- HPOlib/config_parser/parse.py | 78 ++-------- .../smac/smac_2_06_01-dev_parser.py | 35 +---- .../spearmint_april2013_mod_parser.py | 39 +---- .../tpe/hyperopt_august2013_mod_parser.py | 31 +--- HPOlib/wrapping.py | 118 +------------- HPOlib/wrapping_util.py | 145 +++++++++++++++++- tests/testsuite.py | 2 + tests/unittests/__init__.py | 1 + tests/unittests/test_wrapping.py | 88 ----------- tests/unittests/test_wrapping_util.py | 89 +++++++++++ 11 files changed, 264 insertions(+), 366 deletions(-) create mode 100644 tests/unittests/__init__.py create mode 100644 tests/unittests/test_wrapping_util.py diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index b7475bdd..774de6e6 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -22,7 +22,7 @@ import subprocess import sys -from config_parser.parse import parse_config +from HPOlib.wrapping_util import get_configuration logger = logging.getLogger("HPOlib.check_before_start") @@ -91,7 +91,7 @@ def _check_config(experiment_dir): def _check_function(experiment_dir): # Check whether path function exists config_file = os.path.join(experiment_dir, "config.cfg") - config = parse_config(config_file, allow_no_value=True) + config = get_configuration(experiment_dir, None, None) path = config.get("HPOLIB", "function") diff --git a/HPOlib/config_parser/parse.py b/HPOlib/config_parser/parse.py index 8e291426..f0b57355 100644 --- a/HPOlib/config_parser/parse.py +++ b/HPOlib/config_parser/parse.py @@ -26,41 +26,26 @@ logger = logging.getLogger("HPOlib.config_parser.parse") -def parse_config(config_file, allow_no_value=True, optimizer_version="", +def parse_config(config_files, allow_no_value=True, optimizer_version="", cli_values=None): - # Reads config_file - # Overwrites with default values from generalDefault.cfg - # Loads optimizer specific parser, called 'optimizer_version'_parser.py, which can read its own default config - if not os.path.isfile(config_file): - raise Exception('%s is not a valid file\n' % os.path.join( - os.getcwd(), config_file)) + if type(config_files) is str: + if not os.path.isfile(config_files): + raise Exception('%s is not a valid file\n' % os.path.join( + os.getcwd(), config_files)) + else: + for config_file in config_files: + if not os.path.isfile(config_file): + raise Exception('%s is not a valid file\n' % os.path.join( + os.getcwd(), config_file)) config = ConfigParser.SafeConfigParser(allow_no_value=allow_no_value) - config.read(config_file) + config.read(config_files) if cli_values is not None: config.readfp(cli_values) - # Load general default configs - config_fn_default = \ - os.path.join(os.path.dirname(os.path.realpath(__file__)), "generalDefault.cfg") - config_default = ConfigParser.SafeConfigParser(allow_no_value=True) - - if not os.path.isfile(config_fn_default): - raise Exception('%s does not exist\n' % config_fn_default) - config_default.read(config_fn_default) - # -------------------------------------------------------------------------- - # Defaults for HPOLIB + # Check critical values # -------------------------------------------------------------------------- - # Set defaults for section HPOLIB - for option in ('algorithm', 'run_instance', 'numberCV', - 'leading_algo_info', 'numberOfConcurrentJobs', - 'runsolver_time_limit', 'total_time_limit', 'memory_limit', - 'cpu_limit', 'leading_runsolver_info', 'max_crash_per_cv'): - if not config.has_option('HPOLIB', option): - config.set('HPOLIB', option, - config_default.get('HPOLIB', option)) - if not config.has_option('HPOLIB', 'numberOfJobs') or \ config.get('HPOLIB', 'numberOfJobs') == '': raise Exception('numberOfJobs not specified in .cfg') @@ -71,43 +56,4 @@ def parse_config(config_file, allow_no_value=True, optimizer_version="", config.get('HPOLIB', 'function') == '': raise Exception('No function specified in .cfg') - # Load optimizer parsing module - if optimizer_version == "": - return config - optimizer_version_parser = optimizer_version + "_parser" - optimizer_version_parser_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), - optimizer_version_parser + ".py") - # noinspection PyBroadException,PyUnusedLocal - try: - optimizer_module_loaded = imp.load_source(optimizer_version_parser, optimizer_version_parser_path) - except Exception as e: - logger.critical('Could not find\n%s\n\tin\n%s\n\t relative to\n%s' % - (optimizer_version_parser, optimizer_version_parser_path, os.getcwd())) - import traceback - logger.critical(traceback.format_exc()) - sys.exit(1) - - # Add optimizer specific defaults - config = optimizer_module_loaded.add_default(config) - return config - -# TODO: remove, does not seem to be useful anymore -# def main(): -# config_fn = sys.argv[1] -# logger.info('Read config from %s..' % config_fn) -# logger.info('\twith spearmint..',) -# parse_config(config_fn, optimizer_module="spearmint") -# logger.info('\t..finished') -# logger.info('\twith smac..') -# parse_config(config_fn, optimizer_module="smac") -# logger.info('\t..finished') -# logger.info('\twith tpe..') -# parse_config(config_fn, optimizer_module="tpe") -# logger.info('\t..finished') -# logger.info('..finished') -# -# if __name__ == "__main__": -# main() - - diff --git a/HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py b/HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py index 241419f5..620c7d53 100644 --- a/HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py +++ b/HPOlib/optimizers/smac/smac_2_06_01-dev_parser.py @@ -25,38 +25,7 @@ logger = logging.getLogger("HPOlib.optimizers.smac.smac_2_06_01-dev_parser") -def add_default(config): - # This module reads smacDefault.cfg and adds this defaults to a given config - - assert isinstance(config, ConfigParser.RawConfigParser), \ - 'config is not a valid instance' - - # Find out name of .cfg, we are in anything_parser.py[c] - optimizer_config_fn = os.path.splitext(__file__)[0][:-7] + "Default.cfg" - if not os.path.exists(optimizer_config_fn): - logger.critical("No default config %s found" % optimizer_config_fn) - sys.exit(1) - - smac_config = ConfigParser.SafeConfigParser(allow_no_value=True) - smac_config.read(optimizer_config_fn) - # -------------------------------------------------------------------------- - # SMAC - # -------------------------------------------------------------------------- - # Set defaults for SMAC - if not config.has_section('SMAC'): - config.add_section('SMAC') - # optional arguments (exec_dir taken out as is does not seem to be used) - for option in ('numRun', 'instanceFile', 'intraInstanceObj', 'runObj', - 'testInstanceFile', 'p', 'rf_full_tree_bootstrap', - 'rf_split_min', 'adaptiveCapping', 'maxIncumbentRuns', - 'numIterations', 'runtimeLimit', 'deterministic', - 'retryTargetAlgorithmRunCount', - 'intensification_percentage'): - if not config.has_option('SMAC', option): - config.set('SMAC', option, - smac_config.get('SMAC', option)) - - # special cases +def manipulate_config(config): if not config.has_option('SMAC', 'cutoffTime'): config.set('SMAC', 'cutoffTime', str(config.getint('HPOLIB', 'runsolver_time_limit') + 100)) @@ -71,7 +40,7 @@ def add_default(config): config.set('SMAC', 'numConcurrentAlgoExecs', config.get('HPOLIB', 'numberOfConcurrentJobs')) - path_to_optimizer = smac_config.get('SMAC', 'path_to_optimizer') + path_to_optimizer = config.get('SMAC', 'path_to_optimizer') if not os.path.isabs(path_to_optimizer): path_to_optimizer = os.path.join(os.path.dirname(os.path.realpath(__file__)), path_to_optimizer) diff --git a/HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py b/HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py index f8695b21..1dd1b2c0 100644 --- a/HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py +++ b/HPOlib/optimizers/spearmint/spearmint_april2013_mod_parser.py @@ -25,46 +25,19 @@ logger = logging.getLogger("HPOlib.optimizers.spearmint.spearmint_april2013_mod_parser") -def add_default(config): - # This module reads __file__Default.cfg and adds this defaults to a given config - assert isinstance(config, ConfigParser.RawConfigParser), \ - "config is not a valid instance" - - # Find out name of .cfg, we are in anything_parser.py[c] - optimizer_config_fn = os.path.splitext(__file__)[0][:-7] + "Default.cfg" - if not os.path.exists(optimizer_config_fn): - logger.critical("No default config %s found" % optimizer_config_fn) - sys.exit(1) - - spearmint_config = ConfigParser.SafeConfigParser(allow_no_value=True) - spearmint_config.read(optimizer_config_fn) - - # -------------------------------------------------------------------------- - # SPEARMINT - # -------------------------------------------------------------------------- - # Set default for SPEARMINT - if not config.has_section('SPEARMINT'): - config.add_section('SPEARMINT') - # optional arguments - for option in ('script', 'method', 'grid_size', 'config', 'grid_seed', - 'spearmint_polling_time', 'max_concurrent', 'method_args'): - if not config.has_option('SPEARMINT', option): - config.set('SPEARMINT', option, - spearmint_config.get('SPEARMINT', option)) - +def manipulate_config(config): # special cases if not config.has_option('SPEARMINT', 'method'): - config.set('SPEARMINT', 'method', - spearmint_config.get('SPEARMINT', 'method')) - config.set('SPEARMINT', 'method-args', - spearmint_config.get('SPEARMINT', 'method-args')) + raise Exception("SPEARMINT:method not specified in .cfg") + if not config.has_option('SPEARMINT', 'method_args'): + raise Exception("SPEARMINT:method-args not specified in .cfg") # GENERAL if not config.has_option('SPEARMINT', 'max_finished_jobs'): config.set('SPEARMINT', 'max_finished_jobs', config.get('HPOLIB', 'numberOfJobs')) - path_to_optimizer = spearmint_config.get('SPEARMINT', 'path_to_optimizer') + path_to_optimizer = config.get('SPEARMINT', 'path_to_optimizer') if not os.path.isabs(path_to_optimizer): path_to_optimizer = os.path.join(os.path.dirname(os.path.realpath(__file__)), path_to_optimizer) @@ -75,4 +48,4 @@ def add_default(config): config.set('SPEARMINT', 'path_to_optimizer', path_to_optimizer) - return config \ No newline at end of file + return config diff --git a/HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py b/HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py index 8a899b4b..070257f9 100644 --- a/HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py +++ b/HPOlib/optimizers/tpe/hyperopt_august2013_mod_parser.py @@ -25,36 +25,18 @@ logger = logging.getLogger("HPOlib.optimizers.tpe.hyperopt_august2013_mod_parser") -def add_default(config): - # This module reads __file__Default.cfg and adds this defaults to a given config - assert isinstance(config, ConfigParser.RawConfigParser), \ - "config is not a valid instance" - - # Find out name of .cfg, we are in anything_parser.py[c] - optimizer_config_fn = os.path.splitext(__file__)[0][:-7] + "Default.cfg" - if not os.path.exists(optimizer_config_fn): - logger.critical("No default config %s found" % optimizer_config_fn) - sys.exit(1) - - tpe_config = ConfigParser.SafeConfigParser(allow_no_value=True) - tpe_config.read(optimizer_config_fn) - # -------------------------------------------------------------------------- - # TPE - # -------------------------------------------------------------------------- - # Set default for TPE +def manipulate_config(config): if not config.has_section('TPE'): config.add_section('TPE') # optional cases if not config.has_option('TPE', 'space'): - config.set('TPE', 'space', - tpe_config.get('TPE', 'space')) + raise Exception("TPE:space not specified in .cfg") if not config.has_option('TPE', 'numberEvals'): - config.set('TPE', 'numberEvals', - config.get('HPOLIB', 'numberOfJobs')) + config.set('TPE', 'numberEvals', config.get('HPOLIB', 'numberOfJobs')) - path_to_optimizer = tpe_config.get('TPE', 'path_to_optimizer') + path_to_optimizer = config.get('TPE', 'path_to_optimizer') if not os.path.isabs(path_to_optimizer): path_to_optimizer = os.path.join(os.path.dirname(os.path.realpath(__file__)), path_to_optimizer) @@ -65,9 +47,4 @@ def add_default(config): config.set('TPE', 'path_to_optimizer', path_to_optimizer) - # TODO: I don't think we need this anymore - # Anyway set this - # Makes it possible to call, e.g. hyperopt_august2013_mod via simply tpe - # config.set('DEFAULT', 'optimizer_version', tpe_config.get('TPE', 'version')) - return config \ No newline at end of file diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index e864ef79..01e90f22 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -20,15 +20,13 @@ import imp import logging import os -from StringIO import StringIO import subprocess import sys import time -from config_parser.parse import parse_config - import HPOlib import HPOlib.check_before_start as check_before_start +import HPOlib.wrapping_util as wrapping_util __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" @@ -127,89 +125,10 @@ def use_arg_parser(): return args, unknown -def parse_config_values_from_unknown_arguments(unknown_arguments, config): - """Parse values not recognized by use_option_parser for config values. - - Args: - unknown_arguments: A list of arguments which is returned by - use_option_parser. It should only contain keys which are allowed - in config files. - config: A ConfigParser.SafeConfigParser object which contains all keys - should be parsed from the unknown_arguments list. - Returns: - an argparse.Namespace object containing the parsed values. - Raises: - an error if an argument from unknown_arguments is not a key in config - """ - further_possible_command_line_arguments = [] - for section in config.sections(): - for key in config.options(section): - further_possible_command_line_arguments.append("--" + section + - ":" + key) - for key in config.defaults(): - further_possible_command_line_arguments.append("--DEFAULT:" + key) - - parser = ArgumentParser() - for argument in further_possible_command_line_arguments: - parser.add_argument(argument) - - return parser.parse_args(unknown_arguments) - - -def config_with_cli_arguments(config, config_overrides): - arg_dict = vars(config_overrides) - for section in config.sections(): - for key in config.options(section): - cli_key = "%s:%s" % (section, key) - if cli_key in arg_dict: - config.set(section, key, arg_dict[cli_key]) - else: - config.remove_option(section, key) - return config - - -def override_config_with_cli_arguments(config, config_overrides): - arg_dict = vars(config_overrides) - for key in arg_dict: - if arg_dict[key] is not None: - split_key = key.split(":") - if len(split_key) != 2: - raise ValueError("Command line argument must be in the style " - "of --SECTION:key. Please check that there " - "is exactly one colon. You provided: %s" % - key) - if split_key[0] == 'DEFAULT' and split_key[1] == "optimizer_version": - raise NotImplementedError("Overwriting the optimizer_version via a cli is a bad idea." - "Try to do something else!\n" - "You were trying to use %s instead of %s" - % (arg_dict[key], config.get("DEFAULT", "optimizer_version"))) - config.set(split_key[0], split_key[1], arg_dict[key]) - - return config - - -def save_config_to_file(file_handle, config, write_nones=True): - if len(config.defaults()) > 0: - file_handle.write("[DEFAULT]\n") - for key in config.defaults(): - if (config.get("DEFAULT", key) is None and write_nones) or \ - config.get("DEFAULT", key) is not None: - file_handle.write(key + " = " + config.get("DEFAULT", key) + "\n") - - for section in config.sections(): - file_handle.write("[" + section + "]\n") - for key in config.options(section): - if (config.get(section, key) is None and write_nones) or \ - config.get(section, key) is not None: - if config.get(section, key) is None or \ - config.get(section, key) != "": - file_handle.write("%s = %s\n" % (key, config.get(section, key))) - - def main(): """Start an optimization of the HPOlib. For documentation see the comments inside this function and the general HPOlib documentation.""" - args, unknown = use_arg_parser() + args, unknown_arguments = use_arg_parser() if args.working_dir: os.chdir(args.working_dir) @@ -229,36 +148,9 @@ def main(): logger.warning("You called -o %s, I am using optimizer defined in %sDefault.cfg" % (optimizer, optimizer_version)) optimizer = os.path.basename(optimizer_version) - """How the configuration is parsed: - 1. The command line is already parsed, we have a list of unknown arguments - 2. call parse_config to find out all allowed keys. Ignore the values. - 3. call parse_config_values_from_unknown_arguments to get parameters for - the optimization software from the command line in a config object - 4. call parse_config again to fill in the missing values - """ - config_file = os.path.join(experiment_dir, "config.cfg") - # Is the config file really the right place to get the allowed keys for a - # hyperparameter optimizer? - config = parse_config(config_file, allow_no_value=True, optimizer_version=optimizer_version) - - # Parse command line arguments - config_overrides = parse_config_values_from_unknown_arguments(unknown, config) - - # Remove every option from the configuration which was not present on the - # command line - config = config_with_cli_arguments(config, config_overrides) - - # Transform to a string so we can pass it to parse_config - fh = StringIO() - save_config_to_file(fh, config, write_nones=False) - fh.seek(0) - - config = parse_config(config_file, allow_no_value=True, - optimizer_version=optimizer_version, - cli_values=fh) - fh.close() + config = wrapping_util.get_configuration(experiment_dir, + optimizer_version, unknown_arguments) - # config = override_config_with_cli_arguments(config, config_overrides) # Saving the config file is down further at the bottom, as soon as we get # hold of the new optimizer directory wrapping_dir = os.path.dirname(os.path.realpath(__file__)) @@ -290,7 +182,7 @@ def main(): with open(os.path.join(optimizer_dir_in_experiment, "config.cfg"), "w") as f: config.set("HPOLIB", "is_not_original_config_file", "True") - save_config_to_file(f, config) + wrapping_util.save_config_to_file(f, config, write_nones=True) # _check_function check_before_start.check_second(experiment_dir) diff --git a/HPOlib/wrapping_util.py b/HPOlib/wrapping_util.py index ef676cd3..b9863401 100644 --- a/HPOlib/wrapping_util.py +++ b/HPOlib/wrapping_util.py @@ -16,14 +16,18 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from argparse import ArgumentParser +from ConfigParser import SafeConfigParser import datetime import logging +import imp import math import traceback import os +from StringIO import StringIO import sys -from config_parser.parse import parse_config +import config_parser.parse as parse __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" @@ -84,14 +88,147 @@ def load_experiment_config_file(): # Load the config file, this holds information about data, black box fn etc. try: cfg_filename = "config.cfg" - cfg = parse_config(cfg_filename, allow_no_value=True) - if not cfg.has_option("HPOLIB", "is_not_original_config_file"): + config = SafeConfigParser(allow_no_value=True) + config.read(cfg_filename) + if not config.has_option("HPOLIB", "is_not_original_config_file"): logger.critical("Config file in directory %s seems to be an" " original config which was not created by wrapping.py. " "Please contact the HPOlib maintainer to solve this issue.") sys.exit(1) - return cfg + return config except IOError as e: logger.critical("Could not open config file in directory %s" % os.getcwd()) sys.exit(1) + + +def get_configuration(experiment_dir, optimizer_version, unknown_arguments): + """How the configuration is parsed: + 1. The command line is already parsed, we have a list of unknown arguments + 2. Assemble list of all config files related to this experiment, these are + 1. the general default config + 2. the optimizer default config + 3. the experiment configuration + 3. call parse_config to find out all allowed keys. Ignore the values. + 4. call parse_config_values_from_unknown_arguments to get parameters for + the optimization software from the command line in a config object + 5. call parse_config again. Read config files in the order specified + under point 2. Then read the command line arguments. + """ + config_files = list() + general_default = os.path.join(os.path.dirname(parse.__file__), + "generalDefault.cfg") + config_files.append(general_default) + # Load optimizer parsing module + if optimizer_version != "" and optimizer_version is not None: + optimizer_version_parser = optimizer_version + "_parser" + # If optimizer_version_parser is an absolute path, the path of + # __file__ will be ignored + optimizer_version_parser_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + optimizer_version_parser + ".py") + # noinspection PyBroadException,PyUnusedLocal + try: + optimizer_module_parser = imp.load_source(optimizer_version_parser, + optimizer_version_parser_path) + except Exception as e: + logger.critical('Could not find\n%s\n\tin\n%s\n\t relative to\n%s' % + (optimizer_version_parser, + optimizer_version_parser_path, os.getcwd())) + import traceback + + logger.critical(traceback.format_exc()) + sys.exit(1) + + optimizer_config_fn = os.path.splitext(optimizer_module_parser + .__file__)[0][ + :-7] + "Default.cfg" + if not os.path.exists(optimizer_config_fn): + logger.critical("No default config %s found" % optimizer_config_fn) + sys.exit(1) + config_files.append(optimizer_config_fn) + + config_files.append(os.path.join(experiment_dir, "config.cfg")) + # Is the config file really the right place to get the allowed keys for a + # hyperparameter optimizer? + config = parse.parse_config(config_files, allow_no_value=True, + optimizer_version=optimizer_version) + + if unknown_arguments is not None: + # Parse command line arguments + config_overrides = parse_config_values_from_unknown_arguments( + unknown_arguments, config) + # Remove every option from the configuration which was not present on the + # command line + config = config_with_cli_arguments(config, config_overrides) + # Transform to a string so we can pass it to parse_config + fh = StringIO() + save_config_to_file(fh, config, write_nones=False) + fh.seek(0) + else: + fh = None + config = parse.parse_config(config_files, allow_no_value=True, + optimizer_version=optimizer_version, + cli_values=fh) + if fh is not None: + fh.close() + if optimizer_version != "" and optimizer_version is not None: + config = optimizer_module_parser.manipulate_config(config) + return config + + +def parse_config_values_from_unknown_arguments(unknown_arguments, config): + """Parse values not recognized by use_option_parser for config values. + + Args: + unknown_arguments: A list of arguments which is returned by + use_option_parser. It should only contain keys which are allowed + in config files. + config: A ConfigParser.SafeConfigParser object which contains all keys + should be parsed from the unknown_arguments list. + Returns: + an argparse.Namespace object containing the parsed values. + Raises: + an error if an argument from unknown_arguments is not a key in config + """ + further_possible_command_line_arguments = [] + for section in config.sections(): + for key in config.options(section): + further_possible_command_line_arguments.append("--" + section + + ":" + key) + for key in config.defaults(): + further_possible_command_line_arguments.append("--DEFAULT:" + key) + + parser = ArgumentParser() + for argument in further_possible_command_line_arguments: + parser.add_argument(argument) + + return parser.parse_args(unknown_arguments) + + +def config_with_cli_arguments(config, config_overrides): + arg_dict = vars(config_overrides) + for section in config.sections(): + for key in config.options(section): + cli_key = "%s:%s" % (section, key) + if cli_key in arg_dict: + config.set(section, key, arg_dict[cli_key]) + else: + config.remove_option(section, key) + return config + + +def save_config_to_file(file_handle, config, write_nones=True): + if len(config.defaults()) > 0: + file_handle.write("[DEFAULT]\n") + for key in config.defaults(): + if (config.get("DEFAULT", key) is None and write_nones) or \ + config.get("DEFAULT", key) is not None: + file_handle.write(key + " = " + config.get("DEFAULT", key) + "\n") + + for section in config.sections(): + file_handle.write("[" + section + "]\n") + for key in config.options(section): + if (config.get(section, key) is None and write_nones) or \ + config.get(section, key) is not None: + file_handle.write("%s = %s\n" % (key, config.get(section, key))) \ No newline at end of file diff --git a/tests/testsuite.py b/tests/testsuite.py index f049029a..f69f973b 100644 --- a/tests/testsuite.py +++ b/tests/testsuite.py @@ -7,6 +7,7 @@ #import unittests.test_gridsearch as test_gridsearch import unittests.test_runsolver_wrapper as test_runsolver_wrapper import unittests.test_wrapping as test_wrapping +import unittests.test_wrapping_util as test_wrapping_util def suite(): @@ -18,6 +19,7 @@ def suite(): #_suite.addTest(unittest.makeSuite(test_gridsearch.GridSearchTest)) _suite.addTest(unittest.makeSuite(test_runsolver_wrapper.RunsolverWrapperTest)) _suite.addTest(unittest.makeSuite(test_wrapping.WrappingTest)) + _suite.addTest(unittest.makeSuite(test_wrapping_util.WrappingTestUtil)) return _suite diff --git a/tests/unittests/__init__.py b/tests/unittests/__init__.py new file mode 100644 index 00000000..8f0ce6cb --- /dev/null +++ b/tests/unittests/__init__.py @@ -0,0 +1 @@ +__author__ = 'feurerm' diff --git a/tests/unittests/test_wrapping.py b/tests/unittests/test_wrapping.py index 575ca8a4..1b766ab6 100644 --- a/tests/unittests/test_wrapping.py +++ b/tests/unittests/test_wrapping.py @@ -57,94 +57,6 @@ def test_use_option_parser_the_right_way(self): self.assertEqual(args.seed, 1) self.assertEqual(args.title, 'DBNet') self.assertEqual(len(unknown), 0) - - def test_use_option_parser_with_config(self): - sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet', '-o', 'SMAC', - '--HPOLIB:total_time_limit', '3600'] - args, unknown = wrapping.use_arg_parser() - self.assertEqual(len(unknown), 2) - config = parse.parse_config("dummy_config.cfg", allow_no_value=True) - config_args = wrapping.parse_config_values_from_unknown_arguments( - unknown, config) - self.assertEqual(vars(config_args)['HPOLIB:total_time_limit'], - '3600') - - def test_override_config_with_cli_arguments(self): - unknown = ['--HPOLIB:total_time_limit', '50', - '--HPOLIB:numberofjobs', '2'] - config = parse.parse_config("dummy_config.cfg", allow_no_value=True) - config_args = wrapping.parse_config_values_from_unknown_arguments( - unknown, config) - new_config = wrapping.override_config_with_cli_arguments(config, - config_args) - self.assertEqual(new_config.get('HPOLIB', 'numberofjobs'), '2') - - def test_override_config_with_cli_arguments_2(self): - """This tests whether wrapping - .parse_config_values_from_unknown_arguments works as expected. This - test is not sufficient to conclude that we can actually override - optimizer parameters from the command line. We must make sure that - the config parser is invoked before we parse the cli arguments.""" - unknown = ['--HPOLIB:total_time_limit', '50', - '--HPOLIB:numberofjobs', '2', - '--GRIDSEARCH:params', 'test'] - config = parse.parse_config("dummy_config.cfg", allow_no_value=True) - config_args = wrapping.parse_config_values_from_unknown_arguments( - unknown, config) - new_config = wrapping.override_config_with_cli_arguments(config, - config_args) - self.assertEqual(new_config.get('GRIDSEARCH', 'params'), 'test') - - def test_save_config_to_file(self): - unknown = ['--HPOLIB:total_time_limit', '50', - '--HPOLIB:numberofjobs', '2'] - config = parse.parse_config("dummy_config.cfg", allow_no_value=True) - config.set("HPOLIB", "total_time_limit", None) - string_stream = StringIO.StringIO() - wrapping.save_config_to_file(string_stream, config) - file_content = string_stream.getvalue() - asserted_file_content = "[HPOLIB]\n" \ - "numberofjobs = 1\n" \ - "result_on_terminate = 1\n"\ - "function = 1\n"\ - "algorithm = cv.py\n"\ - "run_instance = runsolver_wrapper.py\n"\ - "numbercv = 1\n"\ - "numberofconcurrentjobs = 1\n"\ - "runsolver_time_limit = 3600\n"\ - "total_time_limit = None\n"\ - "memory_limit = 2000\n"\ - "cpu_limit = 14400\n"\ - "max_crash_per_cv = 3\n" \ - "[GRIDSEARCH]\n" \ - "params = params.pcs\n" - - self.assertEqual(asserted_file_content, file_content) - string_stream.close() - - def test_save_config_to_file_ignore_none(self): - config = parse.parse_config("dummy_config.cfg", allow_no_value=True) - config.set("HPOLIB", "total_time_limit", None) - string_stream = StringIO.StringIO() - wrapping.save_config_to_file(string_stream, config, write_nones=False) - file_content = string_stream.getvalue() - asserted_file_content = "[HPOLIB]\n" \ - "numberofjobs = 1\n" \ - "result_on_terminate = 1\n"\ - "function = 1\n"\ - "algorithm = cv.py\n"\ - "run_instance = runsolver_wrapper.py\n"\ - "numbercv = 1\n"\ - "numberofconcurrentjobs = 1\n"\ - "runsolver_time_limit = 3600\n"\ - "memory_limit = 2000\n"\ - "cpu_limit = 14400\n"\ - "max_crash_per_cv = 3\n" \ - "[GRIDSEARCH]\n" \ - "params = params.pcs\n" - - self.assertEqual(asserted_file_content, file_content) - string_stream.close() # General main test @unittest.skip("Not implemented yet") diff --git a/tests/unittests/test_wrapping_util.py b/tests/unittests/test_wrapping_util.py new file mode 100644 index 00000000..d4828a3d --- /dev/null +++ b/tests/unittests/test_wrapping_util.py @@ -0,0 +1,89 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import ConfigParser +import os +import shutil +import sys +import unittest +import tempfile +import StringIO + +import HPOlib.wrapping as wrapping +import HPOlib.wrapping_util as wrapping_util +import HPOlib.config_parser.parse as parse + +class WrappingTestUtil(unittest.TestCase): + def setUp(self): + # Change into the test directory + os.chdir(os.path.dirname(os.path.realpath(__file__))) + + # Make sure there is no config file + try: + os.remove("./config.cfg") + except: + pass + + + def test_save_config_to_file(self): + unknown = ['--HPOLIB:total_time_limit', '50', + '--HPOLIB:numberofjobs', '2'] + config = parse.parse_config("dummy_config.cfg", allow_no_value=True) + config.set("HPOLIB", "total_time_limit", None) + string_stream = StringIO.StringIO() + wrapping_util.save_config_to_file(string_stream, config) + file_content = string_stream.getvalue() + asserted_file_content = "[HPOLIB]\n" \ + "numberofjobs = 1\n" \ + "result_on_terminate = 1\n"\ + "function = 1\n"\ + "total_time_limit = None\n"\ + "[GRIDSEARCH]\n" \ + "params = params.pcs\n" + + self.assertEqual(asserted_file_content, file_content) + string_stream.close() + + def test_save_config_to_file_ignore_none(self): + config = parse.parse_config("dummy_config.cfg", allow_no_value=True) + config.set("HPOLIB", "total_time_limit", None) + string_stream = StringIO.StringIO() + wrapping_util.save_config_to_file(string_stream, config, + write_nones=False) + file_content = string_stream.getvalue() + asserted_file_content = "[HPOLIB]\n" \ + "numberofjobs = 1\n" \ + "result_on_terminate = 1\n"\ + "function = 1\n"\ + "[GRIDSEARCH]\n" \ + "params = params.pcs\n" + + self.assertEqual(asserted_file_content, file_content) + string_stream.close() + + def test_use_option_parser_with_config(self): + sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet', '-o', 'SMAC', + '--HPOLIB:numberofjobs', '2'] + args, unknown = wrapping.use_arg_parser() + self.assertEqual(len(unknown), 2) + config = ConfigParser.SafeConfigParser(allow_no_value=True) + config.read("dummy_config.cfg") + config_args = wrapping_util.parse_config_values_from_unknown_arguments( + unknown, config) + self.assertEqual(vars(config_args)['HPOLIB:numberofjobs'], + '2') From 9315a7e39f694534599bde1e5e6ddd46941983fc Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 5 Mar 2014 09:29:40 +0100 Subject: [PATCH 064/139] FIX: relative path to optimizer is converted to an absolute path. This is done before we (maybe) change the working directory. --- HPOlib/wrapping.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 01e90f22..ac5dd4db 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -130,6 +130,15 @@ def main(): comments inside this function and the general HPOlib documentation.""" args, unknown_arguments = use_arg_parser() + # Convert the path to the optimizer to be an absolute path, which is + # necessary later when we change the working directory + optimizer = args.optimizer + if not os.path.isabs(optimizer): + relative_path = optimizer + optimizer = os.path.abspath(optimizer) + logger.info("Converting relative optimizer path %s to absolute " + "optimizer path %s." % (relative_path, optimizer)) + if args.working_dir: os.chdir(args.working_dir) @@ -141,7 +150,6 @@ def main(): import HPOlib.Experiment as Experiment import HPOlib.wrapping_util - optimizer = args.optimizer # Check how many optimizer versions are present optimizer_version = check_before_start.check_optimizer(optimizer) From bdea6e30082144e546db9ce6e81fe905c6a9a9ee Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 5 Mar 2014 10:43:21 +0100 Subject: [PATCH 065/139] REFACTOR: issue #24 --- HPOlib/check_before_start.py | 47 +++++++------------ HPOlib/wrapping.py | 12 ++--- optimizers/smac/smac_2_06_01-dev.py | 13 +++++ .../spearmint/spearmint_april2013_mod.py | 23 +++++++++ optimizers/tpe/hyperopt_august2013_mod.py | 29 ++++++++++++ 5 files changed, 86 insertions(+), 38 deletions(-) diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index 774de6e6..2257c8e0 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -17,6 +17,7 @@ # along with this program. If not, see . import glob +import imp import logging import os import subprocess @@ -34,13 +35,14 @@ def _check_runsolver(): # check whether runsolver is in path process = subprocess.Popen("which runsolver", stdout=subprocess.PIPE, - stderr=subprocess.PIPE, shell=True, - executable="/bin/bash") + stderr=subprocess.PIPE, shell=True, executable="/bin/bash") stdoutdata, stderrdata = process.communicate() + if stdoutdata is not None and "runsolver" in stdoutdata: pass else: - raise Exception("Runsolver cannot not be found. Are you sure that it's installed?\n" + raise Exception("Runsolver cannot not be found. " + "Are you sure that it's installed?\n" "Your $PATH is: " + os.environ['PATH']) @@ -62,21 +64,12 @@ def _check_modules(): except: raise ImportError("Scipy cannot be imported. Are you sure that it's installed?") - import networkx - import google.protobuf - try: import theano except ImportError: logger.warning("Theano not found. You might need this to run some " "more complex benchmarks!") - try: - import pymongo - from bson.objectid import ObjectId - except ImportError: - raise ImportError("Pymongo cannot be imported. Are you sure it's installed?") - if 'cuda' not in os.environ['PATH']: logger.warning("CUDA not in $PATH") @@ -133,30 +126,19 @@ def check_optimizer(optimizer): if not os.path.exists(version + ".py"): logger.critical("Sorry I cannot find the script to call your optimizer: %s.py" % version) sys.exit(1) - else: - return version - -def check_zeroth(experiment_dir): - logger.info("Check config.cfg..",) - _check_config(experiment_dir) - logger.info("..passed") + # Check the optimizer dependencies + optimizer_module = imp.load_source(version, version + ".py") + optimizer_module.check_dependencies() + return version # noinspection PyUnusedLocal def check_first(experiment_dir): """ Do some checks before optimizer is loaded """ - main() - - -def check_second(experiment_dir): - """ Do remaining tests """ - logger.info("Check algorithm..",) - _check_function(experiment_dir) + logger.info("Check config.cfg..",) + _check_config(experiment_dir) logger.info("..passed") - - -def main(): logger.info("Check dependencies:") logger.info("Runsolver..") _check_runsolver() @@ -166,5 +148,8 @@ def main(): logger.info("..passed") -if __name__ == "__main__": - main() \ No newline at end of file +def check_second(experiment_dir): + """ Do remaining tests """ + logger.info("Check algorithm..",) + _check_function(experiment_dir) + logger.info("..passed") diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index ac5dd4db..94834ec5 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -27,6 +27,7 @@ import HPOlib import HPOlib.check_before_start as check_before_start import HPOlib.wrapping_util as wrapping_util +# Experiment is imported after we check for numpy __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" @@ -143,14 +144,14 @@ def main(): os.chdir(args.working_dir) experiment_dir = os.getcwd() - check_before_start.check_zeroth(experiment_dir) + check_before_start.check_first(experiment_dir) # Now we can safely import non standard things import numpy as np - import HPOlib.Experiment as Experiment - import HPOlib.wrapping_util + import HPOlib.Experiment as Experiment # Wants numpy and scipy - # Check how many optimizer versions are present + # Check how many optimizer versions are present and if all dependencies + # are installed optimizer_version = check_before_start.check_optimizer(optimizer) logger.warning("You called -o %s, I am using optimizer defined in %sDefault.cfg" % (optimizer, optimizer_version)) @@ -167,9 +168,6 @@ def main(): # Try adding runsolver to path os.putenv('PATH', os.environ['PATH'] + ":" + wrapping_dir + "/../runsolver/src/") - # _check_runsolver, _check_modules() - check_before_start.check_first(experiment_dir) - # TODO: We also don't need this # build call cmd = "export PYTHONPATH=$PYTHONPATH:" + wrapping_dir + "\n" diff --git a/optimizers/smac/smac_2_06_01-dev.py b/optimizers/smac/smac_2_06_01-dev.py index d2b66a83..6f16757a 100644 --- a/optimizers/smac/smac_2_06_01-dev.py +++ b/optimizers/smac/smac_2_06_01-dev.py @@ -41,6 +41,19 @@ #optimizer_str = "smac_2_06_01-dev" +def check_dependencies(): + process = subprocess.Popen("which java", stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True, executable="/bin/bash") + stdoutdata, stderrdata = process.communicate() + + if stdoutdata is not None and "java" in stdoutdata: + pass + else: + raise Exception("Java cannot not be found. " + "Are you sure that it's installed?\n" + "Your $PATH is: " + os.environ['PATH']) + + def _get_state_run(optimizer_dir): rungroups = glob.glob(optimizer_dir + "scenario-SMAC*") if len(rungroups) == 1: diff --git a/optimizers/spearmint/spearmint_april2013_mod.py b/optimizers/spearmint/spearmint_april2013_mod.py index 5bf10154..ae08af11 100644 --- a/optimizers/spearmint/spearmint_april2013_mod.py +++ b/optimizers/spearmint/spearmint_april2013_mod.py @@ -38,6 +38,29 @@ "https://github.com/JasperSnoek/spearmint/tree/613350f2f617de3af5f101b1dc5eccf60867f67e") +def check_dependencies(): + try: + import networkx + except ImportError: + raise ImportError("Networkx cannot be imported. Are you sure it's " + "installed?") + try: + import google.protobuf + except ImportError: + raise ImportError("Google protobuf cannot be imported. Are you sure " + "it's installed?") + try: + import numpy + except ImportError: + raise ImportError("Numpy cannot be imported. Are you sure that it's" + " installed?") + try: + import scipy + except ImportError: + raise ImportError("Scipy cannot be imported. Are you sure that it's" + " installed?") + + def build_spearmint_call(config, options, optimizer_dir): print call = 'python ' + os.path.join(config.get('SPEARMINT', 'path_to_optimizer'), 'spearmint_sync.py') diff --git a/optimizers/tpe/hyperopt_august2013_mod.py b/optimizers/tpe/hyperopt_august2013_mod.py index 8d493e25..4c4a67c3 100644 --- a/optimizers/tpe/hyperopt_august2013_mod.py +++ b/optimizers/tpe/hyperopt_august2013_mod.py @@ -30,6 +30,35 @@ __contact__ = "automl.org" +def check_dependencies(): + try: + import nose + except ImportError: + raise ImportError("Nose cannot be imported. Are you sure it's " + "installed?") + try: + import networkx + except ImportError: + raise ImportError("Networkx cannot be imported. Are you sure it's " + "installed?") + try: + import pymongo + from bson.objectid import ObjectId + except ImportError: + raise ImportError("Pymongo cannot be imported. Are you sure it's" + " installed?") + try: + import numpy + except ImportError: + raise ImportError("Numpy cannot be imported. Are you sure that it's" + " installed?") + try: + import scipy + except ImportError: + raise ImportError("Scipy cannot be imported. Are you sure that it's" + " installed?") + + def build_tpe_call(config, options, optimizer_dir): # For TPE we have to cd to the exp_dir call = "cd " + optimizer_dir + "\n" + \ From bf12347ae9b5abbde7f8890ae628c7dc8361330c Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 5 Mar 2014 11:09:04 +0100 Subject: [PATCH 066/139] ENHANCEMENT: choose time measurements The user can now choose whether to use the time measurement of the runsolver or the time returned by the target algorithm. This feature is necessary to do Bayesian Optimization with pre-evaluated grid points. --- HPOlib/config_parser/generalDefault.cfg | 1 + HPOlib/runsolver_wrapper.py | 8 +++++++- benchmarks/branin/branin.py | 2 +- benchmarks/har6/har6.py | 2 +- tests/unittests/test_runsolver_wrapper.py | 2 -- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/HPOlib/config_parser/generalDefault.cfg b/HPOlib/config_parser/generalDefault.cfg index a5ecc885..4945477e 100644 --- a/HPOlib/config_parser/generalDefault.cfg +++ b/HPOlib/config_parser/generalDefault.cfg @@ -2,6 +2,7 @@ #Will be used for wrapping.py and SMAC algorithm = cv.py run_instance = runsolver_wrapper.py +use_own_time_measurement = True numberCV = 1 max_crash_per_cv = 3 diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 02050009..924e4b84 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -236,9 +236,15 @@ def get_parameters(): def parse_output_files(cfg, run_instance_output, runsolver_output_file): - cpu_time, wallclock_time, error = read_runsolver_output( + cpu_time, measured_wallclock_time, error = read_runsolver_output( runsolver_output_file) result_array, result_string = read_run_instance_output(run_instance_output) + instance_wallclock_time = float(result_array[4]) + + if cfg.getboolean("HPOLIB", "use_own_time_measurement") is True: + wallclock_time = measured_wallclock_time + else: + wallclock_time = instance_wallclock_time if error is None and result_string is None: additional_data = "No result string returned. Please have a look " \ diff --git a/benchmarks/branin/branin.py b/benchmarks/branin/branin.py index 416c88fb..1ab2ccfe 100644 --- a/benchmarks/branin/branin.py +++ b/benchmarks/branin/branin.py @@ -62,5 +62,5 @@ def main(params, **kwargs): args, params = benchmark_util.parse_cli() result = main(params, **args) duration = time.time() - starttime - print "Result for ParamILS: %s, %d, 1, %f, %d, %s" % \ + print "Result for ParamILS: %s, %f, 1, %f, %d, %s" % \ ("SAT", abs(duration), result, -1, str(__file__)) diff --git a/benchmarks/har6/har6.py b/benchmarks/har6/har6.py index 4fa3cbae..244848a9 100644 --- a/benchmarks/har6/har6.py +++ b/benchmarks/har6/har6.py @@ -93,5 +93,5 @@ def main(params, **kwargs): args, params = benchmark_util.parse_cli() result = main(params, **args) duration = time.time() - starttime - print "Result for ParamILS: %s, %d, 1, %f, %d, %s" % \ + print "Result for ParamILS: %s, %f, 1, %f, %d, %s" % \ ("SAT", abs(duration), result, -1, str(__file__)) diff --git a/tests/unittests/test_runsolver_wrapper.py b/tests/unittests/test_runsolver_wrapper.py index 0b3a0ed8..bd4621f5 100644 --- a/tests/unittests/test_runsolver_wrapper.py +++ b/tests/unittests/test_runsolver_wrapper.py @@ -101,8 +101,6 @@ def test_read_run_instance_output_result(self): "SAT", "0.35", "1", "0.5", "-1", "Random", "file"]) - - def test_get_trial_index_cv(self): try: os.remove("test_get_trial_index.pkl") From 12758b6bb9f4809b0405b530817e8f4e97c29818 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 5 Mar 2014 14:24:58 +0100 Subject: [PATCH 067/139] FIX #18, #4: setup.py loads optimizer sources setup.py can now download optimizer sources and makes it possible to use HPOlib from the git repo. INSTALL is a quick quide to run a first experiment. --- INSTALL | 152 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 68 +++++++++++-------------- 2 files changed, 181 insertions(+), 39 deletions(-) create mode 100644 INSTALL diff --git a/INSTALL b/INSTALL new file mode 100644 index 00000000..4f8ea4ff --- /dev/null +++ b/INSTALL @@ -0,0 +1,152 @@ +=== INSTALLATION INSTRUCTIONS FOR HPOlib === + +git clone https://github.com/automl/HPOlib.git + +=== Installing inside an virtualenv === +1.) Get virtualenv (http://www.virtualenv.org/en/latest/virtualenv.html#installation) + + pip install virtualenv + +2.) Create and load an virtualenv + + virtualenv virtualHPOlib + source virtualHPOlib/bin/activate + +You're now in an virtual python environment + +3.) Install numpy, scipy, matplotlib, as this doesn't work through setup.py + + easy_install -U distribute + pip install numpy + pip install scipy + pip install matplotlib + +This may take some time. Aftwerwards you can verify having those libs installed with: + + pip freeze + argparse==1.2.1 + backports.ssl-match-hostname==3.4.0.2 + distribute==0.7.3 + matplotlib==1.3.1 + nose==1.3.0 + numpy==1.8.0 + protobuf==2.5.0 + pyparsing==2.0.1 + python-dateutil==2.2 + scipy==0.13.3 + six==1.5.2 + tornado==3.2 + wsgiref==0.1.2 + +4.) run setup.py + + python setup.py install + +This will install HPOlib and some requirements ('networkx', 'protobuf', 'pymongo'). +Be sure your system is connected to the internet, so setup.py can download +optimizer and runsolver code. Your environment now looks like that + + pip freeze + HPOlib==0.0.1 + argparse==1.2.1 + backports.ssl-match-hostname==3.4.0.2 + distribute==0.7.3 + matplotlib==1.3.1 + networkx==1.8.1 + nose==1.3.0 + numpy==1.8.0 + protobuf==2.5.0 + pymongo==2.6.3 + pyparsing==2.0.1 + python-dateutil==2.2 + scipy==0.13.3 + six==1.5.2 + tornado==3.2 + wsgiref==0.1.2 + +and + + ls optimizers/smac + smac_2_06_01-dev_parser.py smac_2_06_01-dev.py smac_2_06_01-dev_src + smac_2_06_01-devDefault.cfg + +5.) You can now run, e.g. smac with 200 evaluations on the branin function: + + cd benchmarks/branin + HPOlib-run -o ../../optimizers/smac/sma -s 23 + +This takes depending on your machine ~2 minutes + + HPOlib-plot FIRSTRUN smac_2_06_01-dev_23_*/smac_*.pkl -s `pwd`/Plots/ + +gives you some nice plots in './Plots'. You can test the other optimizers (spearmint will take quite longer 30min), +but don't be scared by messy output of the optimizer preceded by [ERR]: + + HPOlib-run -o ../../optimizers/tpe/h -s 23 + HPOlib-run -o ../../optimizers/spearmint/s -s 23 + +and again: + + HPOlib-plot SMAC smac_2_06_01-dev_23_*/smac_*.pkl TPE hyperopt_august2013_mod_23_*/hyp*.pkl SPEARMINT spearmint_april2013_mod_23_*/spear*.pkl -s `pwd`/Plots/ +and to check the general performance on this super complex benchmark: + + HPOlib-plot BRANIN smac_2_06_01-dev_23_*/smac_*.pkl hyperopt_august2013_mod_23_*/hyp*.pkl spearmint_april2013_mod_23_*/spear*.pkl -s `pwd`/Plots/ + +=== Using without installation === +If you decide to not install HPOlib, you need to download the optimizer code by yourself + + cd optimizers + wget http://www.automl.org/hyperopt_august2013_mod_src.tar.gz + wget http://www.automl.org/smac_2_06_01-dev_src.tar.gz + wget http://www.automl.org/spearmint_april2013_mod_src.tar.gz + + tar -xf hyperopt_august2013_mod_src.tar.gz + mv hyperopt_august2013_mod_src tpe/ + + tar -xf smac_2_06_01-dev_src.tar.gz + mv smac_2_06_01-dev_src.tar.gz smac/ + + tar -xf spearmint_april2013_mod_src.tar.gz + mv spearmint_april2013_mod_src spearmint/ + + cd ../ + +And you need to install all requirements: + + numpy + matplotlib + networkx + protobuf + scipy + pymongo + +e.g. with + + sudo apt-get install python-numpy python-scipy mongodb python-networkx python-protobuf + +Also you need the runsolver + + wget http://www.cril.univ-artois.fr/~roussel/runsolver/runsolver-3.3.2.tar.bz2 + tar -xf runsolver-3.3.2.tar.bz2 + cd runsolver/src + make + +as this might not work, you can change the makefile via + + sed -i 's/\/usr\/include\/asm\/unistd/\/usr\/include\/unistd/g' ./Makefile + make + +then you need to add runsolver to your PATH + + cd ../../ + export PATH=$PATH:/path/to/runsolver/src/ + +and HPOlib to your PYTHONPATH + + export PYTHONPATH=$PYTHONPATH:`pwd` + +then you can run a benchmark like in step 5.) from installing with setup.py with replacing +''HPOlib-run'' with ''../../scripts/HPOlib-run'' and ''HPOlib-plot'' with ''../../scripts/HPOlib-plot'' + + +== FOR FURTHER DETAILS VISIT: automl.org == \ No newline at end of file diff --git a/setup.py b/setup.py index ce91a156..4b8e9f2d 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ import hashlib from setuptools import setup -from setuptools.command.install import install as install_command +from setuptools.command.install import install import shutil import urllib import os @@ -16,15 +16,8 @@ package_dir = {'HPOlib': 'HPOlib', 'HPOlib.config_parser': 'HPOlib/config_parser', - #'HPOlib.benchmarks': 'HPOlib/benchmarks', - #'HPOlib.optimizers': 'HPOlib/optimizers', - #'HPOlib.benchmarks.branin': 'HPOlib/benchmarks/branin', - #'HPOlib.benchmarks.har6': 'HPOlib/benchmarks/har6', 'HPOlib.Plotting': 'HPOlib/Plotting'} package_data = {'HPOlib.config_parser': ['*.cfg']} - # 'HPOlib': ['*/params.txt', '*/space.py', '*/config.pb'], - # 'HPOlib.benchmarks.branin': ['*.cfg', '*/params.txt', '*/space.py', '*/config.pb'], - # 'HPOlib.benchmarks.har6': ['*.cfg', '*/params.txt', '*/space.py', '*/config.pb']} data_files = [] scripts = ['scripts/HPOlib-run', 'scripts/HPOlib-plot', 'runsolver/src/runsolver'] @@ -37,10 +30,6 @@ def read(fname): def get_find_packages(): packages = ['HPOlib', 'HPOlib.config_parser', - #'HPOlib.benchmarks', - #'HPOlib.optimizers', - #'HPOlib.benchmarks.branin', - #'HPOlib.benchmarks.har6', 'HPOlib.Plotting'] return packages @@ -76,6 +65,7 @@ def copy_folder(new_dir, old_dir): ignore=shutil.ignore_patterns('*.pyc', '*_src*')) except shutil.Error as e: sys.stderr.write("[shutil.ERROR] Could not copy folder from %s to %s\n" % (old_dir, new_dir)) + sys.stderr.write("%s\n" % e.message) return False except Exception, e: sys.stderr.write("[ERROR] Could not copy folder from %s to %s\n" % (old_dir, new_dir)) @@ -96,7 +86,7 @@ def make_dir(new_dir): (e.message, new_dir)) -class AdditionalInstall(install_command): +class AdditionalInstall(install): """Is supposed to install the runsolver, download optimizers, and copy benchmarks""" @@ -205,29 +195,30 @@ def run(self): spearmint = self._copy_and_download_optimizer(optimizer_dir=optimizer_dir, optimizer_name='spearmint', optimizer_tar_name="spearmint_april2013_mod_src.tar.gz", - url="http://www.automl.org/spearmint_april2013_mod_src.tar.gz", + url="http://www.automl.org/spearmint_april2013" + + "_mod_src.tar.gz", md5='aa3eb7961637f9bf2a8cb9b14c4b8791') # Do whatever you want to do - install_command.run(self) + # TODO: Normally one wants to call run(self), but this runs distutils and ignores install_requirements for unknown reasons + # if anyone knows a better way, feel free to change + install.do_egg_install(self) + # Give detailed output to user if not tpe or not smac or not spearmint: sys.stderr.write("[ERROR] Something went wrong while copying and downloading optimizers." + - "Please do the following to be ready to start optimizing:\n\n" + - "rm optimizers\n" + "mkdir optimizers\n" + - "cp HPOlib/optimizers/tpe optimizers/ -r\n" + - "cp HPOlib/optimizers/smac optimizers/ -r\n" + - "cp HPOlib/optimizers/spearmint optimizers/ -r\n" - "cd optimizers\n" + - "wget http://www.automl.org/hyperopt_august2013_mod_src.tar.gz \n" + - "wget http://www.automl.org/smac_2_06_01-dev_src.tar.gz \n" + - "wget http://www.automl.org/spearmint_april2013_mod_src.tar.gz \n" - "tar -xf hyperopt_august2013_mod_src.tar.gz \n" + "mv hyperopt_august2013_mod_src tpe/ \n" + - "tar -xf smac_2_06_01-dev_src.tar.gz \n" + "mv smac_2_06_01-dev_src.tar.gz smac/ \n" + - "tar -xf spearmint_april2013_mod_src.tar.gz \n" + - "mv spearmint_april2013_mod_src.tar.gz spearmint/ \n\n" + - "Thank You!\n") + "Please do the following to be ready to start optimizing:\n\n" + + "cd optimizers\n" + + "wget http://www.automl.org/hyperopt_august2013_mod_src.tar.gz \n" + + "wget http://www.automl.org/smac_2_06_01-dev_src.tar.gz \n" + + "wget http://www.automl.org/spearmint_april2013_mod_src.tar.gz \n" + + "tar -xf hyperopt_august2013_mod_src.tar.gz \n" + + "mv hyperopt_august2013_mod_src tpe/ \n" + + "tar -xf smac_2_06_01-dev_src.tar.gz \n" + "mv smac_2_06_01-dev_src.tar.gz smac/ \n" + + "tar -xf spearmint_april2013_mod_src.tar.gz \n" + + "mv spearmint_april2013_mod_src spearmint/ \n\n" + + "Thank You!\n") if runsolver_needs_to_be_installed and not built: print "[ERROR] Please install runsolver on your own! You can download it from:\n%s%s" % \ ('http://www.cril.univ-artois.fr/~roussel/runsolver/%s', runsolver_tar_name) @@ -237,7 +228,6 @@ def run(self): if built: print "'runsolver' has been downloaded and installed" - setup( name='HPOlib', version=HPOlib.__version__, @@ -246,13 +236,15 @@ def run(self): platforms=['Linux'], author=HPOlib.__authors__, test_suite="tests.testsuite.suite", - install_requires=['argparse>=1.2.1', - 'matplotlib>=1.3.1', - 'networkx>=1.8.1', - 'numpy>=1.8.0', - 'protobuf>=2.5.0', + # setup_requires=['numpy'], + # We require scipy, but this does not automatically install numpy, + # so the user needs to make sure numpy is already installed + install_requires=['argparse','numpy', + 'matplotlib', + 'networkx', + 'protobuf', 'scipy>=0.13.2', - 'pymongo>=2.6.3', + 'pymongo' ], author_email='eggenspk@informatik.uni-freiburg.de', description=desc, @@ -263,9 +255,7 @@ def run(self): package_data=package_data, data_files=data_files, scripts=scripts, - cmdclass={ - 'install': AdditionalInstall, - }, + cmdclass={'install': AdditionalInstall}, classifiers=[ 'Programming Language :: Python :: 2.7', 'Development Status :: 2 - Pre-Alpha', From a4df9d25a08bb2871bef6ada1b8398925453d0cd Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 5 Mar 2014 14:51:48 +0100 Subject: [PATCH 068/139] REFACTOR: optimizer output in wrapping.py --- HPOlib/wrapping.py | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 94834ec5..834db448 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -269,38 +269,34 @@ def main(): logger.info("-----------------------RUNNING----------------------------------") - if args.verbose: - # Print std and err output for optimizer - proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + while proc.poll() is None: logger.info('Optimizer runs with PID (+1): %d' % proc.pid) - while proc.poll() is None: - next_line = proc.stdout.readline() - fh.write(next_line) - sys.stdout.write(next_line) - sys.stdout.flush() + for line in iter(proc.stdout.readline, ''): + fh.write(line) - elif args.silent: - # Print nothing - proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=fh, stderr=fh) - logger.info('Optimizer runs with PID (+1): %d' % proc.pid) - proc.wait() + # Write to stdout only if verbose is on + if args.verbose: + sys.stdout.write(line) + sys.stdout.flush() - else: - # Print only stderr - proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=fh, stderr=subprocess.PIPE) - logger.info('Optimizer runs with PID (+1): %d' % proc.pid) + for line in iter(proc.stderr.readline, ''): + fh.write(line) + + # Write always, except silent is on + if not args.silent: + sys.stderr.write(line) + sys.stderr.flush() - while proc.poll() is None: - next_line = proc.stderr.readline() - fh.write(next_line) - sys.stdout.write("[ERR]:" + next_line) - sys.stdout.flush() + fh.flush() + time.sleep(0.05) ret = proc.returncode - logger.info("\n-----------------------END--------------------------------------") + logger.info("-----------------------END--------------------------------------") fh.close() trials = Experiment.Experiment(optimizer_dir_in_experiment, optimizer) From ebdf2dcbd7184e4a1ef32439eb007700d3bb7a8c Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 5 Mar 2014 15:31:38 +0100 Subject: [PATCH 069/139] FIX: printing of optimizer output is non-blocking --- HPOlib/wrapping.py | 69 +++++++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 834db448..ea29f5af 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -20,8 +20,10 @@ import imp import logging import os +from Queue import Queue, Empty import subprocess import sys +from threading import Thread import time import HPOlib @@ -264,35 +266,58 @@ def main(): logger.info(cmd) output_file = os.path.join(optimizer_dir_in_experiment, optimizer + ".out") fh = open(output_file, "a") - # process = subprocess.Popen(cmd, stdout=fh, stderr=fh, shell=True, executable="/bin/bash") - # ret = process.wait() - - logger.info("-----------------------RUNNING----------------------------------") proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + logger.info("-----------------------RUNNING----------------------------------") + # http://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python + last_output = time.time() + + def enqueue_output(out, queue): + for line in iter(out.readline, b''): + queue.put(line) + out.close() + + stderr_queue = Queue() + stdout_queue = Queue() + stderr_thread = Thread(target=enqueue_output, args=(proc.stderr, stderr_queue)) + stdout_thread = Thread(target=enqueue_output, args=(proc.stdout, stdout_queue)) + stderr_thread.daemon = True + stdout_thread.daemon = True + stderr_thread.start() + stdout_thread.start() + logger.info('Optimizer runs with PID (+1): %d' % proc.pid) while proc.poll() is None: - logger.info('Optimizer runs with PID (+1): %d' % proc.pid) - - for line in iter(proc.stdout.readline, ''): - fh.write(line) - - # Write to stdout only if verbose is on - if args.verbose: - sys.stdout.write(line) - sys.stdout.flush() - - for line in iter(proc.stderr.readline, ''): - fh.write(line) - - # Write always, except silent is on - if not args.silent: - sys.stderr.write(line) - sys.stderr.flush() + try: + for line in stdout_queue.get_nowait(): + fh.write(line) + + # Write to stdout only if verbose is on + if args.verbose: + sys.stdout.write(line) + sys.stdout.flush() + except Empty: + pass + + try: + for line in stderr_queue.get_nowait(): + fh.write(line) + + # Write always, except silent is on + if not args.silent: + sys.stderr.write(line) + sys.stderr.flush() + except Empty: + pass fh.flush() - time.sleep(0.05) + time.sleep(0.25) + + if time.time() - last_output > 1: + sys.stdout.write("1 second\n") + sys.stdout.flush() + last_output = time.time() ret = proc.returncode From 460523117517b14146265fd9b5952825d2f5665b Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 5 Mar 2014 15:38:36 +0100 Subject: [PATCH 070/139] REFACTOR #15: added parameter handles_cv to config SMAC can handle crossvalidation, so it does not need to call cv.py. Nevertheless we want to do bookkeeping and need to know whether HPOlib or something else calls runsolver_wrapper.py If an optimizer prefers to call runsolver_wrapper and does cv on it's own ''handles_cv'' will be set to 1 in optimizerDefault.cfg --- HPOlib/config_parser/generalDefault.cfg | 4 + HPOlib/runsolver_wrapper.py | 88 +++++++++++---------- optimizers/smac/smac_2_06_01-devDefault.cfg | 4 + 3 files changed, 56 insertions(+), 40 deletions(-) diff --git a/HPOlib/config_parser/generalDefault.cfg b/HPOlib/config_parser/generalDefault.cfg index 4945477e..f47bb281 100644 --- a/HPOlib/config_parser/generalDefault.cfg +++ b/HPOlib/config_parser/generalDefault.cfg @@ -19,3 +19,7 @@ cpu_limit = 14400 # E.g. Information for Theano and cuda leading_runsolver_info = leading_algo_info = + +# If an algorithm can handle cv on it's own +# and calls runsolver_wrapper instead of cv.py as a function +handles_cv=0 \ No newline at end of file diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 924e4b84..99c3e687 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -17,7 +17,6 @@ # along with this program. If not, see . from collections import OrderedDict -import imp import logging import numpy as np import os @@ -27,12 +26,11 @@ import time import HPOlib.Experiment as Experiment -import HPOlib.wrapping_util as wrapping_util +import HPOlib.wrapping_util as wrappingUtil __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" - logging.basicConfig(format='[%(levelname)s] [%(asctime)s:%(name)s] %(' 'message)s', datefmt='%H:%M:%S') hpolib_logger = logging.getLogger("HPOlib") @@ -56,30 +54,30 @@ def remove_param_metadata(params): new_name = para if "LOG10_" in para: pos = para.find("LOG10") - new_name = para[0:pos] + para[pos+6:] + new_name = para[0:pos] + para[pos + 6:] # new_name = new_name.strip("_") params[new_name] = np.power(10, float(params[para])) del params[para] elif "LOG2" in para: pos = para.find("LOG2_") - new_name = para[0:pos] + para[pos+5:] + new_name = para[0:pos] + para[pos + 5:] # new_name = new_name.strip("_") params[new_name] = np.power(2, float(params[para])) del params[para] elif "LOG_" in para: pos = para.find("LOG") - new_name = para[0:pos] + para[pos+4:] + new_name = para[0:pos] + para[pos + 4:] # new_name = new_name.strip("_") params[new_name] = np.exp(float(params[para])) del params[para] - #Check for Q value, returns round(x/q)*q + #Check for Q value, returns round(x/q)*q m = re.search(r'Q[0-999\.]{1,10}_', para) if m is not None: pos = new_name.find(m.group(0)) - tmp = new_name[0:pos] + new_name[pos+len(m.group(0)):] + tmp = new_name[0:pos] + new_name[pos + len(m.group(0)):] #tmp = tmp.strip("_") q = float(m.group(0)[1:-1]) - params[tmp] = round(float(params[new_name])/q)*q + params[tmp] = round(float(params[new_name]) / q) * q del params[new_name] @@ -127,7 +125,6 @@ def read_run_instance_output(run_instance_output): """ Read the run_instance output file """ - result_string = None result_array = None fh = open(run_instance_output, "r") run_instance_content = fh.readlines() @@ -135,8 +132,8 @@ def read_run_instance_output(run_instance_output): result_string = None for line in run_instance_content: match = re.search(r"\s*[Rr]esult\s+(?:([Ff]or)|([oO]f))\s" - r"+(?:(HAL)|(ParamILS)|(SMAC)|([tT]his [wW]rapper))", - line) + r"+(?:(HAL)|(ParamILS)|(SMAC)|([tT]his [wW]rapper))", + line) if match: pos = match.start(0) result_string = line[pos:].strip() @@ -157,7 +154,7 @@ def get_trial_index(experiment, fold, params): for idx, trial in enumerate(experiment.trials): exp = trial['params'] if exp == params and (idx, fold) not in experiment.instance_order and \ - (experiment.get_trial_from_id(idx)['instance_results'][fold] == np.NaN or \ + (experiment.get_trial_from_id(idx)['instance_results'][fold] == np.NaN or experiment.get_trial_from_id(idx)['instance_results'][fold] != experiment.get_trial_from_id(idx)['instance_results'][fold]): new = False @@ -191,7 +188,6 @@ def parse_command_line(): def get_function_filename(cfg): - fn_path = None exp_dir = os.path.dirname(os.getcwd()) # Check whether function path is absolute @@ -214,17 +210,16 @@ def make_command(cfg, fold, param_string, run_instance_output): memory_limit = cfg.getint('HPOLIB', 'memory_limit') cpu_limit = cfg.getint('HPOLIB', 'cpu_limit') fn_filename = get_function_filename(cfg) - python_cmd = cfg.get("HPOLIB", "leading_algo_info") + " python " + \ - fn_filename + " --fold %d --folds %d --params %s" % \ - (fold, cfg.getint("HPOLIB", "numberCV"), param_string) + python_cmd = cfg.get("HPOLIB", "leading_algo_info") + " python " + fn_filename + python_cmd += " --fold %d --folds %d --params %s" % (fold, cfg.getint("HPOLIB", "numberCV"), param_string) # Do not write the actual task in quotes because runsolver will not work - # then; also we need use-pty antd timestamp so that the "solver" output + # then; also we need use-pty and timestamp so that the "solver" output # is flushed to the output directory delay = 0 - cmd = cfg.get("HPOLIB", "leading_runsolver_info") + \ - " runsolver -o %s --timestamp --use-pty -W %d -C %d -M %d -d %d %s" \ - % (run_instance_output, time_limit, cpu_limit, memory_limit, delay, - python_cmd) + cmd = cfg.get("HPOLIB", "leading_runsolver_info") + cmd += " runsolver -o %s --timestamp --use-pty -W %d -C %d -M %d -d %d %s" \ + % (run_instance_output, time_limit, cpu_limit, memory_limit, delay, + python_cmd) return cmd @@ -250,21 +245,21 @@ def parse_output_files(cfg, run_instance_output, runsolver_output_file): additional_data = "No result string returned. Please have a look " \ "at " + run_instance_output rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("HPOLIB", - "result_on_terminate"), additional_data) + "result_on_terminate"), additional_data) os.remove(runsolver_output_file) elif error is None and result_array[3] != "SAT": - additional_data = "Please have a look at " + run_instance_output + "."\ - "The output status is not \"SAT\"" + additional_data = "Please have a look at " + run_instance_output + "." \ + "The output status is not \"SAT\"" rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("HPOLIB", - "result_on_terminate"), additional_data) + "result_on_terminate"), additional_data) os.remove(runsolver_output_file) elif error is None and not np.isfinite(float(result_array[6].strip(","))): additional_data = "Response value is not finite. Please have a look " \ "at " + run_instance_output rval = (cpu_time, wallclock_time, "UNSAT", cfg.getfloat("HPOLIB", - "result_on_terminate"), additional_data) + "result_on_terminate"), additional_data) elif error is None: # Remove the run_instance_output only if there is a valid result @@ -275,8 +270,9 @@ def parse_output_files(cfg, run_instance_output, runsolver_output_file): else: rval = (cpu_time, wallclock_time, "CRASHED", cfg.getfloat("HPOLIB", - "result_on_terminate"), error + " Please have a look at " + - runsolver_output_file) + "result_on_terminate"), + error + " Please have a look at " + + runsolver_output_file) # It is useful to have the run_instance_output for debugging os.remove(run_instance_output) @@ -285,29 +281,40 @@ def parse_output_files(cfg, run_instance_output, runsolver_output_file): def format_return_string(status, runtime, runlength, quality, seed, additional_data): - return_string = "Result for ParamILS: %s, %f, %d, %f, %d, %s" %\ - (status, runtime, runlength, quality, seed, additional_data) + return_string = "Result for ParamILS: %s, %f, %d, %f, %d, %s" % \ + (status, runtime, runlength, quality, seed, additional_data) return return_string def main(): - optimizer = get_optimizer() + """ + If we are not called from cv means we are called from CLI. This means + the optimizer itself handles crossvalidation (smac). To keep a nice .pkl we have to do some + bookkeeping here + """ + + cfg = wrappingUtil.load_experiment_config_file() + called_from_cv = True + if cfg.getint('HPOLIB', 'handles_cv') == 1: + # If Our Optimizer can handle crossvalidation, + # we are called from CLI. To keep a sane nice .pkl + # we have to do some bookkeeping here + called_from_cv = False + # This has to be done here for SMAC, since smac does not call cv.py - if 'smac' in optimizer: + if not called_from_cv: cv_starttime = time.time() experiment = load_experiment_file() experiment.start_cv(cv_starttime) del experiment - cfg = wrapping_util.load_experiment_config_file() - fold, seed = parse_command_line() # Side-effect: removes all additional information like log and applies # transformations to the parameters params = get_parameters() param_string = " ".join([key + " " + params[key] for key in params]) - time_string = wrapping_util.get_time_string() + time_string = wrappingUtil.get_time_string() run_instance_output = os.path.join(os.getcwd(), time_string + "_run_instance.out") runsolver_output_file = os.path.join(os.getcwd(), @@ -324,7 +331,7 @@ def main(): process = subprocess.Popen(cmd, stdout=fh, stderr=fh, shell=True, executable="/bin/bash") - + logger.info("-----------------------RUNNING RUNSOLVER----------------------------") process.wait() fh.close() @@ -343,12 +350,12 @@ def main(): else: # TODO: We need a global stopping mechanism pass - del experiment #release lock + del experiment # release lock return_string = format_return_string(status, wallclock_time, 1, result, seed, additional_data) - if 'smac' in optimizer: + if not called_from_cv: experiment = load_experiment_file() experiment.end_cv(time.time()) del experiment @@ -356,6 +363,7 @@ def main(): logger.info(return_string) print return_string return return_string - + + if __name__ == "__main__": main() diff --git a/optimizers/smac/smac_2_06_01-devDefault.cfg b/optimizers/smac/smac_2_06_01-devDefault.cfg index 65c31f67..d5577506 100644 --- a/optimizers/smac/smac_2_06_01-devDefault.cfg +++ b/optimizers/smac/smac_2_06_01-devDefault.cfg @@ -1,3 +1,7 @@ +[HPOLIB] +# Because smac does intensify and does not need to run all k-folds +handles_cv=1 + [SMAC] # cutoffTime = runsolver_time_limit + 100 sec From c6890ac5411595dcc640f1d285be76c92fdf2cc9 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 5 Mar 2014 16:18:13 +0100 Subject: [PATCH 071/139] REFACTOR #29: replace (job_id, params) by (*args, **kwargs) cv.main works now for tpe and spearmint as it accepts *args and **kwargs instead of job_id and params. Tpe also does not need to call an extra method cv.doForTpe cv.main can be called with any arguments. Any named argument 'params' or the first dict in arg will be handled as the former 'params' --- HPOlib/cv.py | 71 ++++++++++++++++++++++----------------- optimizers/tpe/tpecall.py | 6 ++-- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/HPOlib/cv.py b/HPOlib/cv.py index 231bdd64..2d4b99a7 100644 --- a/HPOlib/cv.py +++ b/HPOlib/cv.py @@ -45,13 +45,13 @@ def get_optimizer(): return "_".join(os.getcwd().split("/")[-1].split("_")[0:-2]) -def loadExperimentFile(): +def load_experiment_file(): optimizer = get_optimizer() experiment = Experiment(".", optimizer) return experiment -def doCV(params, folds=10): +def do_cv(params, folds=10): logger.info("Starting Cross validation") sys.stdout.flush() optimizer = get_optimizer() @@ -60,15 +60,15 @@ def doCV(params, folds=10): # Now evaluate $fold times # TODO This is a bad idea, no one replaces this by the worst possible # value later on - if folds < 1: return np.NaN + if folds < 1: + return np.NaN # Store the results to hand them back to tpe and spearmint results = [] try: logger.info("%s" % params) - param_array = ["-" + str(param_name) + " " + str(params[param_name]) \ - for param_name in params] + param_array = ["-" + str(param_name) + " " + str(params[param_name]) for param_name in params] param_string = " ".join(param_array) for fold in range(folds): @@ -78,15 +78,13 @@ def doCV(params, folds=10): # Cutofftime, cutofflength and seed can be safely ignored since they # are read in runsolver_wrapper runsolver_wrapper_script = "python " + \ - os.path.join(os.path.dirname(os.path.realpath(__file__)), \ - "runsolver_wrapper.py") + os.path.join(os.path.dirname(os.path.realpath(__file__)), "runsolver_wrapper.py") cmd = "%s %d %s %d %d %d %s" % \ - (runsolver_wrapper_script, fold, optimizer, 0, 0, 0, \ - param_string) + (runsolver_wrapper_script, fold, optimizer, 0, 0, 0, param_string) logger.info("Calling command:\n%s" % cmd) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, shell=True, executable="/bin/bash") + stderr=subprocess.PIPE, shell=True, executable="/bin/bash") logger.info("--------------RUNNING RUNSOLVER_WRAPPER--------------") stdoutdata, stderrdata = process.communicate() if stdoutdata: @@ -105,7 +103,7 @@ def doCV(params, folds=10): results.append(float(result_array[6].strip(","))) break - if result_string == None: + if result_string is None: raise NotImplementedError("No result string available or result string doesn't contain SAT") # If a specified number of runs crashed, quit the whole cross validation @@ -113,8 +111,8 @@ def doCV(params, folds=10): worst_possible = cfg.getfloat("HPOLIB", "result_on_terminate") crashed_runs = np.nansum([0 if res != worst_possible else 1 for res in results]) if crashed_runs >= cfg.getint("HPOLIB", "max_crash_per_cv"): - logger.warning("Aborting CV because the number of crashes " \ - "excceds the configured max_crash_per_cv value") + logger.warning("Aborting CV because the number of crashes " + "exceeds the configured max_crash_per_cv value") return worst_possible # TODO: Error Handling @@ -126,7 +124,7 @@ def doCV(params, folds=10): logger.error(format_traceback(sys.exc_info())) logger.error("CV failed %s %s" % (sys.exc_info()[0], e)) # status = "CRASHED" - status = "SAT" + # status = "SAT" mean = np.NaN # Do not return any kind of nan because this would break spearmint @@ -153,8 +151,8 @@ def flatten_parameter_dict(params): # This class enables us to distinguish tuples, which are parameters from # tuples which contain different things... class Parameter: - def __init__(self, param): - self.param = param + def __init__(self, pparam): + self.pparam = pparam params_to_check = list() params_to_check.append(params) @@ -164,7 +162,6 @@ def __init__(self, param): param = params_to_check.pop() if type(param) in (list, tuple, np.ndarray): - added_literal = False for sub_param in param: params_to_check.append(sub_param) @@ -173,12 +170,12 @@ def __init__(self, param): params_to_check.extend([Parameter(tmp_param) for tmp_param in zip(param.keys(), param.values())]) elif isinstance(param, Parameter): - key = param.param[0] - value = param.param[1] + key = param.pparam[0] + value = param.pparam[1] if type(value) == dict: params_to_check.append(value) elif type(value) in (list, tuple, np.ndarray) and \ - all([type(v) not in (list, tuple, np.ndarray) for v in value]): + all([type(v) not in (list, tuple, np.ndarray) for v in value]): # Spearmint special case, keep only the first element new_dict[key] = value[0] elif type(value) in (list, tuple, np.ndarray): @@ -193,17 +190,31 @@ def __init__(self, param): return params -def main(job_id, params): - # Forget job_id which comes from spearmint +def main(*args, **kwargs): + logger.critical('args: %s kwargs: %s' % (str(args), str(kwargs))) + + params = None + + if 'params' in kwargs.keys(): + params = kwargs['params'] + else: + for arg in args: + if type(arg) == dict: + params = arg + + if params is None: + logger.critical("No parameter dict found in cv.py.\n" + "args: %s\n kwargs: %s" % (args, kwargs)) + sys.exit(1) # Load the experiment to do time-keeping cv_starttime = time.time() - experiment = loadExperimentFile() + experiment = load_experiment_file() # experiment.next_iteration() experiment.start_cv(cv_starttime) del experiment - cfg_filename = "config.cfg" + # cfg_filename = "config.cfg" cfg = load_experiment_config_file() # Load number of folds @@ -211,19 +222,19 @@ def main(job_id, params): params = flatten_parameter_dict(params) - res = doCV(params, folds=folds) + res = do_cv(params, folds=folds) logger.info("Result: %f" % res) # Load the experiment to do time-keeping - experiment = loadExperimentFile() + experiment = load_experiment_file() experiment.end_cv(time.time()) del experiment return res -def doForTPE(params): - return main(42, params) +#def doForTPE(params): +# return main(42, params) def read_params_from_command_line(): @@ -249,5 +260,5 @@ def read_params_from_command_line(): if __name__ == "__main__": - params = read_params_from_command_line() - main(42, params) + cli_params = read_params_from_command_line() + main(params=cli_params) diff --git a/optimizers/tpe/tpecall.py b/optimizers/tpe/tpecall.py index 8a8b1e41..08d187d7 100644 --- a/optimizers/tpe/tpecall.py +++ b/optimizers/tpe/tpecall.py @@ -20,11 +20,11 @@ from argparse import ArgumentParser import cPickle -from optparse import OptionParser from functools import partial from importlib import import_module import logging import os +import sys import hyperopt @@ -89,15 +89,13 @@ def main(): space, ext = os.path.splitext(os.path.basename(args.spaceFile)) # Then load dict searchSpace and out function cv.py - import sys sys.path.append("./") sys.path.append("") - print os.getcwd() module = import_module(space) search_space = module.space fn = import_module(algo) - fn = fn.doForTPE + fn = fn.main # doForTPE if args.random: # We use a random search From 7ce55d81c72ed3af0c981e6201f38639ac53ccfb Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 5 Mar 2014 16:47:55 +0100 Subject: [PATCH 072/139] REFACTOR: Updatet har6 benchmark --- benchmarks/har6/{tpe => random_hyperopt_august2013_mod} | 0 benchmarks/har6/smac | 1 - benchmarks/har6/spearmint | 1 - 3 files changed, 2 deletions(-) rename benchmarks/har6/{tpe => random_hyperopt_august2013_mod} (100%) delete mode 120000 benchmarks/har6/smac delete mode 120000 benchmarks/har6/spearmint diff --git a/benchmarks/har6/tpe b/benchmarks/har6/random_hyperopt_august2013_mod similarity index 100% rename from benchmarks/har6/tpe rename to benchmarks/har6/random_hyperopt_august2013_mod diff --git a/benchmarks/har6/smac b/benchmarks/har6/smac deleted file mode 120000 index 97e5766b..00000000 --- a/benchmarks/har6/smac +++ /dev/null @@ -1 +0,0 @@ -smac_2_06_01-dev/ \ No newline at end of file diff --git a/benchmarks/har6/spearmint b/benchmarks/har6/spearmint deleted file mode 120000 index 525a4844..00000000 --- a/benchmarks/har6/spearmint +++ /dev/null @@ -1 +0,0 @@ -spearmint_april2013_mod/ \ No newline at end of file From 4514596922845b9b2e1a400f088235b4a1d41331 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 5 Mar 2014 17:29:34 +0100 Subject: [PATCH 073/139] REFACTOR: issue #7 better optimization output This might put heavy load on the HPOlib, we must have a look at this --- HPOlib/Experiment.py | 15 ++++---- HPOlib/wrapping.py | 40 +++++++++++++++++---- optimizers/smac/smac_2_06_01-dev.py | 3 +- optimizers/smac/smac_2_06_01-devDefault.cfg | 1 + 4 files changed, 45 insertions(+), 14 deletions(-) diff --git a/HPOlib/Experiment.py b/HPOlib/Experiment.py index 814062c3..cdf3f504 100644 --- a/HPOlib/Experiment.py +++ b/HPOlib/Experiment.py @@ -25,7 +25,7 @@ import numpy as np -# import HPOlib.Locker as Locker +import HPOlib.Locker as Locker import HPOlib.wrapping_util as wrapping_util @@ -52,12 +52,12 @@ def __init__(self, expt_dir, expt_name, max_wallclock_time= folds = 1 self.jobs_pkl = os.path.join(expt_dir, expt_name + ".pkl") - #self.locker = Locker.Locker() + self.locker = Locker.Locker() # Only one process at a time is allowed to have access to this. #logger.info("Waiting to lock experiments file " + # self.jobs_pkl + "...") - #self.locker.lock_wait(self.jobs_pkl) + self.locker.lock_wait(self.jobs_pkl) #logger.info("...acquired\n") # Does this exist already? @@ -124,10 +124,11 @@ def create_trial(self): def __del__(self): self._save_jobs() - #if self.locker.unlock(self.jobs_pkl): - # sys.stderr.write("Released lock on job grid.\n") - #else: - # raise Exception("Could not release lock on job grid.\n") + if self.locker.unlock(self.jobs_pkl): + pass + # sys.stderr.write("Released lock on job grid.\n") + else: + raise Exception("Could not release lock on job grid.\n") def result_array(self): result = np.array([trial['result'] for trial in self.trials]) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index ea29f5af..7d917bb8 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -272,6 +272,10 @@ def main(): logger.info("-----------------------RUNNING----------------------------------") # http://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python last_output = time.time() + printed_start_configuration = list() + printed_end_configuration = list() + current_best = -1 + minimal_runs_to_go = 2 def enqueue_output(out, queue): for line in iter(out.readline, b''): @@ -288,7 +292,7 @@ def enqueue_output(out, queue): stdout_thread.start() logger.info('Optimizer runs with PID (+1): %d' % proc.pid) - while proc.poll() is None: + while minimal_runs_to_go > 0: # Think of this as a do-while loop... try: for line in stdout_queue.get_nowait(): fh.write(line) @@ -312,13 +316,37 @@ def enqueue_output(out, queue): pass fh.flush() - time.sleep(0.25) - - if time.time() - last_output > 1: - sys.stdout.write("1 second\n") - sys.stdout.flush() + #time.sleep(0.05) + + if not (args.verbose or args.silent) and time.time() - last_output > 1: + trials = Experiment.Experiment(optimizer_dir_in_experiment, + optimizer) + + for i in range(len(printed_end_configuration), len(trials.instance_order)): + if i + 1 > len(printed_start_configuration): + output = "Starting configuration %5d, fold %2d" %\ + (trials.instance_order[i][0], + trials.instance_order[i][1]) + logger.info(output) + printed_start_configuration.append(i) + + if np.isfinite(trials.trials[i]["instance_results"][ + trials.instance_order[i][1]]): + last_result = trials.trials[i]["instance_results"][ + trials.instance_order[i][1]] + tmp_current_best = trials.get_arg_best() + if tmp_current_best <= i: + current_best = tmp_current_best + logger.info("Result %10f, current best %10f" % + (last_result, trials.trials[current_best]["result"])) + printed_end_configuration.append(i) + + del trials last_output = time.time() + if proc.poll() is not None: # the end condition for the + minimal_runs_to_go -= 1 # do-while loop + ret = proc.returncode logger.info("-----------------------END--------------------------------------") diff --git a/optimizers/smac/smac_2_06_01-dev.py b/optimizers/smac/smac_2_06_01-dev.py index 6f16757a..b07693e9 100644 --- a/optimizers/smac/smac_2_06_01-dev.py +++ b/optimizers/smac/smac_2_06_01-dev.py @@ -106,7 +106,8 @@ def build_smac_call(config, options, optimizer_dir): '--save-runs-every-iteration true', '--intensification-percentage', config.get('SMAC', 'intensification_percentage'), - '--rf-split-min', config.get('SMAC', 'rf_split_min')]) + '--rf-split-min', config.get('SMAC', 'rf_split_min'), + '--validation', config.get('SMAC', 'validation')]) if config.getboolean('SMAC', 'deterministic'): call = " ".join([call, '--deterministic true']) diff --git a/optimizers/smac/smac_2_06_01-devDefault.cfg b/optimizers/smac/smac_2_06_01-devDefault.cfg index 65c31f67..f0c01bc8 100644 --- a/optimizers/smac/smac_2_06_01-devDefault.cfg +++ b/optimizers/smac/smac_2_06_01-devDefault.cfg @@ -26,6 +26,7 @@ runtimeLimit = 2147483647 deterministic = True retryTargetAlgorithmRunCount = 0 intensification_percentage = 0 +validation = false # either relative to __file__ or absolute path_to_optimizer = ./smac_2_06_01-dev_src From 265bc7af7de08a02b64c79ebc75c6c70962e4b76 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 5 Mar 2014 17:51:51 +0100 Subject: [PATCH 074/139] renamed README, INSTALLATION to .md files --- INSTALL => INSTALL.md | 0 README => README.md | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename INSTALL => INSTALL.md (100%) rename README => README.md (100%) diff --git a/INSTALL b/INSTALL.md similarity index 100% rename from INSTALL rename to INSTALL.md diff --git a/README b/README.md similarity index 100% rename from README rename to README.md From 2cff53c0754207797eed2495209fbf9d1b80761a Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 5 Mar 2014 18:01:08 +0100 Subject: [PATCH 075/139] FIX: adapt setup.py to changed README name --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4b8e9f2d..490ae1ed 100644 --- a/setup.py +++ b/setup.py @@ -248,7 +248,7 @@ def run(self): ], author_email='eggenspk@informatik.uni-freiburg.de', description=desc, - long_description=read("README"), + long_description=read("README.md"), keywords=keywords, packages=get_find_packages(), package_dir=package_dir, From a4935905d26bfee751eb3309d9aa0ac1987e027f Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 5 Mar 2014 18:11:47 +0100 Subject: [PATCH 076/139] REFACTOR: Removed code smells --- HPOlib/Experiment.py | 6 +-- HPOlib/Locker.py | 7 ++-- HPOlib/check_before_start.py | 18 +++++---- HPOlib/config_parser/parse.py | 2 - HPOlib/cv.py | 12 +++--- HPOlib/data_util.py | 21 ++++++----- HPOlib/runsolver_wrapper.py | 4 +- HPOlib/wrapping.py | 37 ++++++++++--------- HPOlib/wrapping_util.py | 10 ++--- .../spearmint_april2013_mod_parser.py | 2 - 10 files changed, 61 insertions(+), 58 deletions(-) diff --git a/HPOlib/Experiment.py b/HPOlib/Experiment.py index cdf3f504..d6085c73 100644 --- a/HPOlib/Experiment.py +++ b/HPOlib/Experiment.py @@ -280,13 +280,13 @@ def check_cv_finished(self, _id): # parameters. Useful to delete all unnecessary entries after a crash in order # to restart def remove_all_but_first_runs(self, restored_runs): - logger.info("Restored runs %d" % restored_runs) - logger.info("%s %s" % (self.instance_order, len(self.instance_order))) + logger.info("Restored runs %d", restored_runs) + logger.info("%s %s" ,self.instance_order, len(self.instance_order)) if len(self.instance_order) == restored_runs: pass else: for _id, instance in self.instance_order[-1:restored_runs - 1:-1]: - logger.info("Deleting %d %d" % (_id, instance)) + logger.info("Deleting %d %d", _id, instance) if np.isfinite(self.trials[_id]['instance_durations'][instance]): self.total_wallclock_time -= \ self.trials[_id]['instance_durations'][instance] diff --git a/HPOlib/Locker.py b/HPOlib/Locker.py index 72315da1..e80b198a 100755 --- a/HPOlib/Locker.py +++ b/HPOlib/Locker.py @@ -21,7 +21,6 @@ # along with this program. If not, see . import logging import os -import sys import time @@ -49,7 +48,7 @@ def lock(self, filename): self.locks[filename] += 1 return True else: - cmd = 'ln -s /dev/null "%s.lock" 2> /dev/null' % (filename) + cmd = 'ln -s /dev/null "%s.lock" 2> /dev/null' % filename fail = os.system(cmd) if not fail: self.locks[filename] = 1 @@ -57,12 +56,12 @@ def lock(self, filename): def unlock(self, filename): if not self.locks.has_key(filename): - logger.info("Trying to unlock not-locked file %s.\n" % filename) + logger.info("Trying to unlock not-locked file %s.\n", filename) return True if self.locks[filename] == 1: success = safe_delete('%s.lock' % (filename)) if not success: - logger.log("Could not unlock file: %s.\n" % filename) + logger.log("Could not unlock file: %s.\n", filename) del self.locks[filename] return success else: diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index 2257c8e0..980280a8 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -52,7 +52,7 @@ def _check_modules(): import numpy if numpy.__version__ < "1.6.0": logger.warning("WARNING: You are using a numpy %s < 1.6.0. This " - "might not work" % numpy.__version__) + "might not work", numpy.__version__) except: raise ImportError("Numpy cannot be imported. Are you sure that it's installed?") @@ -60,7 +60,7 @@ def _check_modules(): import scipy if scipy.__version__ < "0.12.0": logger.warning("WARNING: You are using a scipy %s < 0.12.0. " - "This might not work" % scipy.__version__) + "This might not work", scipy.__version__) except: raise ImportError("Scipy cannot be imported. Are you sure that it's installed?") @@ -104,7 +104,8 @@ def check_optimizer(optimizer): # Now check how many versions are present parser = glob.glob(os.path.join(path, '*_parser.py')) if len(parser) > 1: - logger.critical("Sorry I don't know which optimizer to use: %s" % parser) + logger.critical("Sorry I don't know which optimizer to use: %s", + parser) sys.exit(1) version = parser[0][:-10] elif len(glob.glob(path + '*_parser.py')) == 1: @@ -113,18 +114,21 @@ def check_optimizer(optimizer): elif len(glob.glob(path + '*_parser.py')) > 1: # Note this is a different case # User told us e.g. "tpe/hyperopt_august" but this was not specific enough - logger.critical("Sorry I don't know which optimizer to use: %s" % glob.glob(path + '*_parser.py')) + logger.critical("Sorry I don't know which optimizer to use: %s", + glob.glob(path + '*_parser.py')) sys.exit(1) else: - logger.critical("We cannot find: %s" % path) + logger.critical("We cannot find: %s", path) sys.exit(1) # Now check the other stuff if not os.path.exists(version + "Default.cfg"): - logger.critical("Sorry I cannot find the default config for your optimizer: %sDefault.cfg" % version) + logger.critical("Sorry I cannot find the default config for your " + "optimizer: %sDefault.cfg", version) sys.exit(1) if not os.path.exists(version + ".py"): - logger.critical("Sorry I cannot find the script to call your optimizer: %s.py" % version) + logger.critical("Sorry I cannot find the script to call your " + "optimizer: %s.py", version) sys.exit(1) # Check the optimizer dependencies diff --git a/HPOlib/config_parser/parse.py b/HPOlib/config_parser/parse.py index f0b57355..a1649167 100644 --- a/HPOlib/config_parser/parse.py +++ b/HPOlib/config_parser/parse.py @@ -18,8 +18,6 @@ import os import logging -import imp -import sys import ConfigParser diff --git a/HPOlib/cv.py b/HPOlib/cv.py index 2d4b99a7..d7a0f7f3 100644 --- a/HPOlib/cv.py +++ b/HPOlib/cv.py @@ -67,7 +67,7 @@ def do_cv(params, folds=10): results = [] try: - logger.info("%s" % params) + logger.info("%s", params) param_array = ["-" + str(param_name) + " " + str(params[param_name]) for param_name in params] param_string = " ".join(param_array) @@ -81,7 +81,7 @@ def do_cv(params, folds=10): os.path.join(os.path.dirname(os.path.realpath(__file__)), "runsolver_wrapper.py") cmd = "%s %d %s %d %d %d %s" % \ (runsolver_wrapper_script, fold, optimizer, 0, 0, 0, param_string) - logger.info("Calling command:\n%s" % cmd) + logger.info("Calling command:\n%s", cmd) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, executable="/bin/bash") @@ -122,7 +122,7 @@ def do_cv(params, folds=10): except Exception as e: logger.error(format_traceback(sys.exc_info())) - logger.error("CV failed %s %s" % (sys.exc_info()[0], e)) + logger.error("CV failed %s %s", sys.exc_info()[0], e) # status = "CRASHED" # status = "SAT" mean = np.NaN @@ -191,7 +191,7 @@ def __init__(self, pparam): def main(*args, **kwargs): - logger.critical('args: %s kwargs: %s' % (str(args), str(kwargs))) + logger.critical('args: %s kwargs: %s', str(args), str(kwargs)) params = None @@ -204,7 +204,7 @@ def main(*args, **kwargs): if params is None: logger.critical("No parameter dict found in cv.py.\n" - "args: %s\n kwargs: %s" % (args, kwargs)) + "args: %s\n kwargs: %s", args, kwargs) sys.exit(1) # Load the experiment to do time-keeping @@ -223,7 +223,7 @@ def main(*args, **kwargs): params = flatten_parameter_dict(params) res = do_cv(params, folds=folds) - logger.info("Result: %f" % res) + logger.info("Result: %f", res) # Load the experiment to do time-keeping experiment = load_experiment_file() diff --git a/HPOlib/data_util.py b/HPOlib/data_util.py index 1dd555a6..a979895f 100644 --- a/HPOlib/data_util.py +++ b/HPOlib/data_util.py @@ -35,7 +35,7 @@ def load_file(filename, file_format, use_percentage): raise IOError("File %s not found", filename) if file_format == "gfile": - logger.info("Loading file: %s" % filename) + logger.info("Loading file: %s", filename) fh = gfile(filename, "rb") data = cPickle.load(fh) if use_percentage >= 100.: @@ -44,10 +44,11 @@ def load_file(filename, file_format, use_percentage): max_data = int(len(data) / 100. * use_percentage) data = data[:max_data] fh.close() - logger.info("Done loading file: %s has %d datapoints" % (filename, len(data))) + logger.info("Done loading file: %s has %d datapoints", filename, + len(data)) elif file_format == "pickle": - logger.info("Loading file: %s" % filename) + logger.info("Loading file: %s", filename) fh = open(filename, "r") data = cPickle.load(fh) if use_percentage >= 100.: @@ -55,10 +56,11 @@ def load_file(filename, file_format, use_percentage): else: data = data[:len(data) / 100. * use_percentage] fh.close() - logger.info("Done loading file: %s has %d datapoints" % (filename, len(data))) + logger.info("Done loading file: %s has %d datapoints", filename, + len(data)) elif file_format == "numpy": - logger.info("Loading file: %s" % filename) + logger.info("Loading file: %s", filename) fh = open(filename, "r") data = np.load(fh) if use_percentage >= 100.: @@ -66,7 +68,8 @@ def load_file(filename, file_format, use_percentage): else: data = data[:len(data) / 100. * use_percentage] fh.close() - logger.info("Done loading file: %s has %d datapoints" % (filename, len(data))) + logger.info("Done loading file: %s has %d datapoints", filename, + len(data)) else: raise ValueError("%s is an unknown training_data_format", file_format) @@ -108,7 +111,7 @@ def prepare_cv_for_fold(data, fold, folds): """ logger.info("Splitting data:") - logger.info("Fold %s of %s folds" % (fold, folds)) + logger.info("Fold %s of %s folds", fold, folds) # Create an array with the split points if data is not None: data_len = len(data) @@ -134,8 +137,8 @@ def prepare_cv_for_fold(data, fold, folds): cv_split_mask[splits[i]:splits[i+1]] = 0 train = data[cv_split_mask] valid = data[~cv_split_mask] - logger.info("%d, %d, %d, %d" % (data.shape, train.shape, valid.shape, - train.itemsize)) + logger.info("%d, %d, %d, %d", data.shape, train.shape, valid.shape, + train.itemsize) else: train = [] valid = [] diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 99c3e687..51084589 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -198,8 +198,8 @@ def get_function_filename(cfg): fn_path = os.path.join(exp_dir, cfg.get("HPOLIB", "function")) if not os.path.exists(fn_path): - logger.critical("%s is not absolute nor in %s. Function cannot be found" % - (cfg.get("HPOLIB", "function"), exp_dir)) + logger.critical("%s is not absolute nor in %s. Function cannot be " + "found", cfg.get("HPOLIB", "function"), exp_dir) sys.exit(1) return fn_path diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 7d917bb8..16bd4f58 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -140,7 +140,7 @@ def main(): relative_path = optimizer optimizer = os.path.abspath(optimizer) logger.info("Converting relative optimizer path %s to absolute " - "optimizer path %s." % (relative_path, optimizer)) + "optimizer path %s.", relative_path, optimizer) if args.working_dir: os.chdir(args.working_dir) @@ -156,7 +156,8 @@ def main(): # are installed optimizer_version = check_before_start.check_optimizer(optimizer) - logger.warning("You called -o %s, I am using optimizer defined in %sDefault.cfg" % (optimizer, optimizer_version)) + logger.warning("You called -o %s, I am using optimizer defined in " + "%sDefault.cfg", optimizer, optimizer_version) optimizer = os.path.basename(optimizer_version) config = wrapping_util.get_configuration(experiment_dir, @@ -179,7 +180,7 @@ def main(): optimizer_dir = os.path.dirname(os.path.realpath(optimizer_version)) optimizer_module = imp.load_source(optimizer_dir, optimizer_version + ".py") except (ImportError, IOError): - logger.critical("Optimizer module %s not found" % optimizer) + logger.critical("Optimizer module %s not found", optimizer) import traceback logger.critical(traceback.format_exc()) sys.exit(1) @@ -237,12 +238,12 @@ def main(): optimizer_dir=optimizer_dir_in_experiment, cmd=cmd) except: - logger.critical("Could not restore runs for %s" % args.restore) + logger.critical("Could not restore runs for %s", args.restore) import traceback logger.critical(traceback.format_exc()) sys.exit(1) - logger.info("Restored %d runs" % restored_runs) + logger.info("Restored %d runs", restored_runs) trials.remove_all_but_first_runs(restored_runs) fh = open(os.path.join(optimizer_dir_in_experiment, optimizer + ".out"), "a") fh.write("#" * 80 + "\n" + "Restart! Restored %d runs.\n" % restored_runs) @@ -290,7 +291,7 @@ def enqueue_output(out, queue): stdout_thread.daemon = True stderr_thread.start() stdout_thread.start() - logger.info('Optimizer runs with PID (+1): %d' % proc.pid) + logger.info('Optimizer runs with PID (+1): %d', proc.pid) while minimal_runs_to_go > 0: # Think of this as a do-while loop... try: @@ -324,10 +325,9 @@ def enqueue_output(out, queue): for i in range(len(printed_end_configuration), len(trials.instance_order)): if i + 1 > len(printed_start_configuration): - output = "Starting configuration %5d, fold %2d" %\ - (trials.instance_order[i][0], - trials.instance_order[i][1]) - logger.info(output) + logger.info("Starting configuration %5d, fold %2d", + trials.instance_order[i][0], + trials.instance_order[i][1]) printed_start_configuration.append(i) if np.isfinite(trials.trials[i]["instance_results"][ @@ -337,8 +337,9 @@ def enqueue_output(out, queue): tmp_current_best = trials.get_arg_best() if tmp_current_best <= i: current_best = tmp_current_best - logger.info("Result %10f, current best %10f" % - (last_result, trials.trials[current_best]["result"])) + logger.info("Result %10f, current best %10f", + last_result, + trials.trials[current_best]["result"]) printed_end_configuration.append(i) del trials @@ -364,17 +365,17 @@ def enqueue_output(out, queue): try: for starttime, endtime in zip(trials.starttime, trials.endtime): total_time += endtime - starttime - logger.info("Needed a total of %f seconds" % total_time) - logger.info("The optimizer %s took %10.5f seconds" %\ - (optimizer, float(calculate_optimizer_time(trials)))) - logger.info("The overhead of HPOlib is %f seconds" % \ - (calculate_wrapping_overhead(trials))) + logger.info("Needed a total of %f seconds", total_time) + logger.info("The optimizer %s took %10.5f seconds", + optimizer, float(calculate_optimizer_time(trials))) + logger.info("The overhead of HPOlib is %f seconds", + calculate_wrapping_overhead(trials)) logger.info("The benchmark itself took %f seconds" % \ trials.total_wallclock_time) except Exception as e: logger.error(HPOlib.wrapping_util.format_traceback(sys.exc_info())) logger.error("Experiment itself went fine, but calculating " - "durations of optimization failed: %s %s" % + "durations of optimization failed: %s %s", sys.exc_info()[0], e) del trials logger.info("Finished with return code: " + str(ret)) diff --git a/HPOlib/wrapping_util.py b/HPOlib/wrapping_util.py index b9863401..f7336628 100644 --- a/HPOlib/wrapping_util.py +++ b/HPOlib/wrapping_util.py @@ -97,7 +97,7 @@ def load_experiment_config_file(): sys.exit(1) return config except IOError as e: - logger.critical("Could not open config file in directory %s" % + logger.critical("Could not open config file in directory %s", os.getcwd()) sys.exit(1) @@ -132,9 +132,9 @@ def get_configuration(experiment_dir, optimizer_version, unknown_arguments): optimizer_module_parser = imp.load_source(optimizer_version_parser, optimizer_version_parser_path) except Exception as e: - logger.critical('Could not find\n%s\n\tin\n%s\n\t relative to\n%s' % - (optimizer_version_parser, - optimizer_version_parser_path, os.getcwd())) + logger.critical('Could not find\n%s\n\tin\n%s\n\t relative to\n%s', + optimizer_version_parser, + optimizer_version_parser_path, os.getcwd()) import traceback logger.critical(traceback.format_exc()) @@ -144,7 +144,7 @@ def get_configuration(experiment_dir, optimizer_version, unknown_arguments): .__file__)[0][ :-7] + "Default.cfg" if not os.path.exists(optimizer_config_fn): - logger.critical("No default config %s found" % optimizer_config_fn) + logger.critical("No default config %s found", optimizer_config_fn) sys.exit(1) config_files.append(optimizer_config_fn) diff --git a/optimizers/spearmint/spearmint_april2013_mod_parser.py b/optimizers/spearmint/spearmint_april2013_mod_parser.py index 1dd1b2c0..b36a004f 100644 --- a/optimizers/spearmint/spearmint_april2013_mod_parser.py +++ b/optimizers/spearmint/spearmint_april2013_mod_parser.py @@ -20,8 +20,6 @@ import os import sys -import ConfigParser - logger = logging.getLogger("HPOlib.optimizers.spearmint.spearmint_april2013_mod_parser") From 71f2e9b6db86dff1fecbff26c0c3f72e8dcab143 Mon Sep 17 00:00:00 2001 From: Katharina Eggensperger Date: Wed, 5 Mar 2014 18:16:23 +0100 Subject: [PATCH 077/139] Update INSTALL.md --- INSTALL.md | 184 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 103 insertions(+), 81 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 4f8ea4ff..31f6173b 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,100 +1,117 @@ -=== INSTALLATION INSTRUCTIONS FOR HPOlib === - -git clone https://github.com/automl/HPOlib.git - -=== Installing inside an virtualenv === -1.) Get virtualenv (http://www.virtualenv.org/en/latest/virtualenv.html#installation) - - pip install virtualenv - -2.) Create and load an virtualenv - - virtualenv virtualHPOlib - source virtualHPOlib/bin/activate - -You're now in an virtual python environment - -3.) Install numpy, scipy, matplotlib, as this doesn't work through setup.py - - easy_install -U distribute - pip install numpy - pip install scipy - pip install matplotlib - -This may take some time. Aftwerwards you can verify having those libs installed with: - - pip freeze - argparse==1.2.1 - backports.ssl-match-hostname==3.4.0.2 - distribute==0.7.3 - matplotlib==1.3.1 - nose==1.3.0 - numpy==1.8.0 - protobuf==2.5.0 - pyparsing==2.0.1 - python-dateutil==2.2 - scipy==0.13.3 - six==1.5.2 - tornado==3.2 - wsgiref==0.1.2 +INSTALLATION INSTRUCTIONS FOR HPOlib +------- + +``git clone https://github.com/automl/HPOlib.git`` + +Installing inside an virtualenv +----------- + +1.) Get [virtualenv](http://www.virtualenv.org/en/latest/virtualenv.html#installation), then load a freshly created virtualenv. (If you are not familiar with virtualenv, you might want to read [more](http://www.virtualenv.org/en/latest/virtualenv.html) about it) + +```bash +pip install virtualenv +virtualenv virtualHPOlib +source virtualHPOlib/bin/activate +``` + +3.) Install ``numpy``, ``scipy``, ``matplotlib``, as this doesn't work through setup.py +```bash +easy_install -U distribute +pip install numpy +pip install scipy +pip install matplotlib +``` +This may take some time. Afterwards you can verify having those libs installed with: + +```bash +pip freeze + argparse==1.2.1 + backports.ssl-match-hostname==3.4.0.2 + distribute==0.7.3 + matplotlib==1.3.1 + nose==1.3.0 + numpy==1.8.0 + pyparsing==2.0.1 + python-dateutil==2.2 + scipy==0.13.3 + six==1.5.2 + tornado==3.2 + wsgiref==0.1.2 +``` 4.) run setup.py +```python python setup.py install +``` -This will install HPOlib and some requirements ('networkx', 'protobuf', 'pymongo'). -Be sure your system is connected to the internet, so setup.py can download +This will install HPOlib and some requirements (``networkx``, ``protobuf``, ``pymongo``). +Be sure your system is **connected to the internet**, so ``setup.py`` can download optimizer and runsolver code. Your environment now looks like that - pip freeze - HPOlib==0.0.1 - argparse==1.2.1 - backports.ssl-match-hostname==3.4.0.2 - distribute==0.7.3 - matplotlib==1.3.1 - networkx==1.8.1 - nose==1.3.0 - numpy==1.8.0 - protobuf==2.5.0 - pymongo==2.6.3 - pyparsing==2.0.1 - python-dateutil==2.2 - scipy==0.13.3 - six==1.5.2 - tornado==3.2 - wsgiref==0.1.2 +```bash +pip freeze + HPOlib==0.0.1 + argparse==1.2.1 + backports.ssl-match-hostname==3.4.0.2 + distribute==0.7.3 + matplotlib==1.3.1 + networkx==1.8.1 + nose==1.3.0 + numpy==1.8.0 + protobuf==2.5.0 + pymongo==2.6.3 + pyparsing==2.0.1 + python-dateutil==2.2 + scipy==0.13.3 + six==1.5.2 + tornado==3.2 + wsgiref==0.1.2 +``` and - ls optimizers/smac - smac_2_06_01-dev_parser.py smac_2_06_01-dev.py smac_2_06_01-dev_src - smac_2_06_01-devDefault.cfg +```bash +ls optimizers/smac + smac_2_06_01-dev_parser.py smac_2_06_01-dev.py smac_2_06_01-dev_src smac_2_06_01-devDefault.cfg +``` 5.) You can now run, e.g. smac with 200 evaluations on the branin function: - cd benchmarks/branin - HPOlib-run -o ../../optimizers/smac/sma -s 23 +```bash +cd benchmarks/branin +HPOlib-run -o ../../optimizers/smac/sma -s 23 +``` -This takes depending on your machine ~2 minutes +This takes depending on your machine ~2 minutes. You can now plot the results of your first experiment: - HPOlib-plot FIRSTRUN smac_2_06_01-dev_23_*/smac_*.pkl -s `pwd`/Plots/ +```bash +HPOlib-plot FIRSTRUN smac_2_06_01-dev_23_*/smac_*.pkl -s `pwd`/Plots/ +``` -gives you some nice plots in './Plots'. You can test the other optimizers (spearmint will take quite longer 30min), -but don't be scared by messy output of the optimizer preceded by [ERR]: +You can test the other optimizers (spearmint will take quite longer 30min): +```bash HPOlib-run -o ../../optimizers/tpe/h -s 23 HPOlib-run -o ../../optimizers/spearmint/s -s 23 +``` and again: +```bash HPOlib-plot SMAC smac_2_06_01-dev_23_*/smac_*.pkl TPE hyperopt_august2013_mod_23_*/hyp*.pkl SPEARMINT spearmint_april2013_mod_23_*/spear*.pkl -s `pwd`/Plots/ and to check the general performance on this super complex benchmark: HPOlib-plot BRANIN smac_2_06_01-dev_23_*/smac_*.pkl hyperopt_august2013_mod_23_*/hyp*.pkl spearmint_april2013_mod_23_*/spear*.pkl -s `pwd`/Plots/ +``` + +Using without installation +----------- + -=== Using without installation === If you decide to not install HPOlib, you need to download the optimizer code by yourself +```bash cd optimizers wget http://www.automl.org/hyperopt_august2013_mod_src.tar.gz wget http://www.automl.org/smac_2_06_01-dev_src.tar.gz @@ -110,43 +127,48 @@ If you decide to not install HPOlib, you need to download the optimizer code by mv spearmint_april2013_mod_src spearmint/ cd ../ +``` And you need to install all requirements: - numpy - matplotlib - networkx - protobuf - scipy - pymongo +* numpy +* matplotlib +* networkx +* protobuf +* scipy +* pymongo e.g. with +```bash sudo apt-get install python-numpy python-scipy mongodb python-networkx python-protobuf +``` Also you need the runsolver +```bash wget http://www.cril.univ-artois.fr/~roussel/runsolver/runsolver-3.3.2.tar.bz2 tar -xf runsolver-3.3.2.tar.bz2 cd runsolver/src make +``` as this might not work, you can change the makefile via +```bash sed -i 's/\/usr\/include\/asm\/unistd/\/usr\/include\/unistd/g' ./Makefile make +``` -then you need to add runsolver to your PATH +then you need to add runsolver (and HPOlib) to your PATH (PYTHONPATH): +```bash cd ../../ export PATH=$PATH:/path/to/runsolver/src/ - -and HPOlib to your PYTHONPATH - export PYTHONPATH=$PYTHONPATH:`pwd` +``` then you can run a benchmark like in step 5.) from installing with setup.py with replacing -''HPOlib-run'' with ''../../scripts/HPOlib-run'' and ''HPOlib-plot'' with ''../../scripts/HPOlib-plot'' - +``HPOlib-run`` with ``../../scripts/HPOlib-run`` and ``HPOlib-plot`` with ``../../scripts/HPOlib-plot`` -== FOR FURTHER DETAILS VISIT: automl.org == \ No newline at end of file +**FOR FURTHER DETAILS VISIT:** www.automl.org From 8b1023c6a8e9b1c6a22fb05438df66e5b43f8dd0 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 6 Mar 2014 10:37:38 +0100 Subject: [PATCH 078/139] FIX: reduced load of polling stderr and stdout --- HPOlib/wrapping.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 16bd4f58..98e90cee 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -295,7 +295,8 @@ def enqueue_output(out, queue): while minimal_runs_to_go > 0: # Think of this as a do-while loop... try: - for line in stdout_queue.get_nowait(): + while True: + line = stdout_queue.get_nowait() fh.write(line) # Write to stdout only if verbose is on @@ -306,18 +307,20 @@ def enqueue_output(out, queue): pass try: - for line in stderr_queue.get_nowait(): + while True: + line = stderr_queue.get_nowait() fh.write(line) # Write always, except silent is on if not args.silent: - sys.stderr.write(line) + sys.stderr.write("[ERR]:" + line) sys.stderr.flush() except Empty: pass fh.flush() - #time.sleep(0.05) + # necessary, otherwise HPOlib-run takes 100% of one processor + time.sleep(0.1) if not (args.verbose or args.silent) and time.time() - last_output > 1: trials = Experiment.Experiment(optimizer_dir_in_experiment, From 6d0a72bc019030b0df226682a8904e59a647c053 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 6 Mar 2014 10:39:02 +0100 Subject: [PATCH 079/139] FIX: added MANIFEST.in, md files included in sdist --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..91f371be --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include *.md \ No newline at end of file From 205f1b66e4b1c90fa7ed1df74134484f469ce8f8 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 6 Mar 2014 10:42:39 +0100 Subject: [PATCH 080/139] REFACTOR: issue #18 Moved the benchmark functions to a file called benchmark_functions. --- HPOlib/benchmark_functions.py | 109 ++++++++++++++++++++++++++++++++++ benchmarks/branin/branin.py | 29 +-------- benchmarks/har6/config.cfg | 2 +- benchmarks/har6/har6.py | 62 ++----------------- 4 files changed, 116 insertions(+), 86 deletions(-) create mode 100644 HPOlib/benchmark_functions.py diff --git a/HPOlib/benchmark_functions.py b/HPOlib/benchmark_functions.py new file mode 100644 index 00000000..1543b4d3 --- /dev/null +++ b/HPOlib/benchmark_functions.py @@ -0,0 +1,109 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from numpy import pi, cos, ndarray, array, exp + + +def save_branin(params, **kwargs): + if "x" not in params or "y" not in params: + raise ValueError("No params found ['x', 'y']\n") + x = float(params["x"]) + y = float(params["y"]) + if type(x) == ndarray or type(y) == ndarray: + x = x[0] + y = y[0] + + if -5 > x or x > 10: + raise ValueError("X value not in between -5 and 10") + if 0 > y or y > 15: + raise ValueError("Y value not in between 0 and 15") + return branin(x, y) + + +def branin(x, y): + """Branin test function + + The number of variables n = 2. + constraints: + -5 <= x <= 10, 0 <= y <= 15 + three global optima: (-pi, 12.275), (pi, 2.275), (9.42478, 2.475), where + branin = 0.397887""" + + result = (y-(5.1/(4*pi**2))*x**2+5*x/pi-6)**2 + result += 10*(1-1/(8*pi))*cos(x)+10 + return result + + +def save_har6(params, **kwargs): + if "x" not in params or "y" not in params or "z" not in params \ + or "a" not in params or "b" not in params or "c" not in params: + sys.stderr.write("No params found ['x', 'y']\n") + return NaN + x = float(params["x"]) + y = float(params["y"]) + z = float(params["z"]) + a = float(params["a"]) + b = float(params["b"]) + c = float(params["c"]) + if type(x) == ndarray: + x = x[0] + y = y[0] + z = z[0] + a = a[0] + b = b[0] + c = c[0] + return har6(x, y, z, a, b, c) + + +def har6(x, y, z, a, b, c): + """6d Hartmann test function + + constraints: + 0 <= xi <= 1, i = 1..6 + global optimum at (0.20169, 0.150011, 0.476874, 0.275332, 0.311652, 0.6573), + where har6 = -3.32236""" + + value = array([x, y, z, a, b, c]) + + if 0 > x or x > 1 or 0 > y or y > 1 or 0 > z or z > 1: + raise ValueError("x=%s, y=%s or z=%s not in between 0 and 1" % + (x, y, z)) + if 0 > a or a > 1 or 0 > b or b > 1 or 0 > c or c > 1: + raise ValueError("a=%s, b=%s or c=%s not in between 0 and 1" % + (a, b, c)) + + a = array([[10.0, 3.0, 17.0, 3.5, 1.7, 8.0], + [ 0.05, 10.0, 17.0, 0.1, 8.0, 14.0], + [ 3.0, 3.5, 1.7, 10.0, 17.0, 8.0], + [17.0, 8.0, 0.05, 10.0, 0.1, 14.0]]) + c = array([1.0, 1.2, 3.0, 3.2]) + p = array([[0.1312, 0.1696, 0.5569, 0.0124, 0.8283, 0.5886], + [0.2329, 0.4135, 0.8307, 0.3736, 0.1004, 0.9991], + [0.2348, 0.1451, 0.3522, 0.2883, 0.3047, 0.6650], + [0.4047, 0.8828, 0.8732, 0.5743, 0.1091, 0.0381]]) + s = 0 + for i in [0,1,2,3]: + sm = a[i,0]*(value[0]-p[i,0])**2 + sm += a[i,1]*(value[1]-p[i,1])**2 + sm += a[i,2]*(value[2]-p[i,2])**2 + sm += a[i,3]*(value[3]-p[i,3])**2 + sm += a[i,4]*(value[4]-p[i,4])**2 + sm += a[i,5]*(value[5]-p[i,5])**2 + s += c[i]*exp(-sm) + result = -s + return result \ No newline at end of file diff --git a/benchmarks/branin/branin.py b/benchmarks/branin/branin.py index 1ab2ccfe..304c1851 100644 --- a/benchmarks/branin/branin.py +++ b/benchmarks/branin/branin.py @@ -18,42 +18,17 @@ import HPOlib.benchmark_util as benchmark_util +import HPOlib.benchmark_functions as benchmark_functions -from numpy import pi, cos, ndarray import time __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" -def branin(params, **kwargs): -# branin function -# The number of variables n = 2. -# constraints: -# -5 <= x <= 10, 0 <= y <= 15 -# three global optima: (-pi, 12.275), (pi, 2.275), (9.42478, 2.475), where -# branin = 0.397887 - if "x" not in params or "y" not in params: - raise ValueError("No params found ['x', 'y']\n") - x = float(params["x"]) - y = float(params["y"]) - if type(x) == ndarray or type(y) == ndarray: - x = x[0] - y = y[0] - - if -5 > x or x > 10: - raise ValueError("X value not in between -5 and 10") - if 0 > y or y > 15: - raise ValueError("Y value not in between 0 and 15") - - result = (y-(5.1/(4*pi**2))*x**2+5*x/pi-6)**2 - result += 10*(1-1/(8*pi))*cos(x)+10 - return result - - def main(params, **kwargs): print 'Params: ', params, - y = branin(params, **kwargs) + y = benchmark_functions.save_branin(params, **kwargs) print 'Result: ', y return y diff --git a/benchmarks/har6/config.cfg b/benchmarks/har6/config.cfg index e9e90e72..f2643e39 100644 --- a/benchmarks/har6/config.cfg +++ b/benchmarks/har6/config.cfg @@ -1,5 +1,5 @@ [SMAC] -p = smac/params.txt +p = smac_2_06_01-dev/params.txt [HPOLIB] function = har6.py diff --git a/benchmarks/har6/har6.py b/benchmarks/har6/har6.py index 244848a9..f7054f6e 100644 --- a/benchmarks/har6/har6.py +++ b/benchmarks/har6/har6.py @@ -16,73 +16,19 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from numpy import NaN, array, ndarray, exp - -import HPOlib.benchmark_util as benchmark_util import sys import time +import HPOlib.benchmark_util as benchmark_util +import HPOlib.benchmark_functions as benchmark_functions + __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" - - -def har6(params, **kwargs): -# 6d Hartmann function -# constraints: -# 0 <= xi <= 1, i = 1..6 -# global optimum at (0.20169, 0.150011, 0.476874, 0.275332, 0.311652, 0.6573), -# where har6 = -3.32236 - if "x" not in params or "y" not in params or "z" not in params \ - or "a" not in params or "b" not in params or "c" not in params: - sys.stderr.write("No params found ['x', 'y']\n") - return NaN - x = float(params["x"]) - y = float(params["y"]) - z = float(params["z"]) - a = float(params["a"]) - b = float(params["b"]) - c = float(params["c"]) - if type(x) == ndarray: - x = x[0] - y = y[0] - z = z[0] - a = a[0] - b = b[0] - c = c[0] - value = array([x, y, z, a, b, c]) - - if 0 > x or x > 1 or 0 > y or y > 1 or 0 > z or z > 1: - raise ValueError("x=%s, y=%s or z=%s not in between 0 and 1" % - (x, y, z)) - if 0 > a or a > 1 or 0 > b or b > 1 or 0 > c or c > 1: - raise ValueError("a=%s, b=%s or c=%s not in between 0 and 1" % - (a, b, c)) - - a = array([[10.0, 3.0, 17.0, 3.5, 1.7, 8.0], - [ 0.05, 10.0, 17.0, 0.1, 8.0, 14.0], - [ 3.0, 3.5, 1.7, 10.0, 17.0, 8.0], - [17.0, 8.0, 0.05, 10.0, 0.1, 14.0]]) - c = array([1.0, 1.2, 3.0, 3.2]) - p = array([[0.1312, 0.1696, 0.5569, 0.0124, 0.8283, 0.5886], - [0.2329, 0.4135, 0.8307, 0.3736, 0.1004, 0.9991], - [0.2348, 0.1451, 0.3522, 0.2883, 0.3047, 0.6650], - [0.4047, 0.8828, 0.8732, 0.5743, 0.1091, 0.0381]]) - s = 0 - for i in [0,1,2,3]: - sm = a[i,0]*(value[0]-p[i,0])**2 - sm += a[i,1]*(value[1]-p[i,1])**2 - sm += a[i,2]*(value[2]-p[i,2])**2 - sm += a[i,3]*(value[3]-p[i,3])**2 - sm += a[i,4]*(value[4]-p[i,4])**2 - sm += a[i,5]*(value[5]-p[i,5])**2 - s += c[i]*exp(-sm) - result = -s - return result def main(params, **kwargs): print 'Params: ', params, - y = har6(params, **kwargs) + y = benchmark_functions.save_har6(params, **kwargs) print 'Result: ', y return y From 7f5abba5057cd338e17743b5807afe77b3b915f2 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Thu, 6 Mar 2014 10:48:33 +0100 Subject: [PATCH 081/139] FIX imports --- HPOlib/benchmark_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/HPOlib/benchmark_functions.py b/HPOlib/benchmark_functions.py index 1543b4d3..675e04a0 100644 --- a/HPOlib/benchmark_functions.py +++ b/HPOlib/benchmark_functions.py @@ -16,7 +16,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from numpy import pi, cos, ndarray, array, exp +from numpy import pi, cos, ndarray, array, exp, NaN +import sys def save_branin(params, **kwargs): From 48c79312b030fce84eb9a4752fa39e3f175381ce Mon Sep 17 00:00:00 2001 From: Katharina Eggensperger Date: Thu, 6 Mar 2014 15:25:10 +0100 Subject: [PATCH 082/139] Update INSTALL.md Added a "problems during installation section" to collect possible errors and solutions, which might happen during running setup.py --- INSTALL.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index 31f6173b..da2fcbaf 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -172,3 +172,11 @@ then you can run a benchmark like in step 5.) from installing with setup.py with ``HPOlib-run`` with ``../../scripts/HPOlib-run`` and ``HPOlib-plot`` with ``../../scripts/HPOlib-plot`` **FOR FURTHER DETAILS VISIT:** www.automl.org + + +**Problems during installation** + +``python setup.py`` crashes with ``ImportError: cannot import name Feature`` during installing pymongo. This happens due to pymongo using a deprecated feature ''Feature'', which is not available in the setuptools version (>2.2). This error is fixed, but not yet available on PYPI. + +Solution: Downgrade ``setuptools`` with ``pip install setuptools==2.2`` and try again or install ``pymongo`` manually. + From f2474a213b6e1f3214963478c0615c98a60f1b97 Mon Sep 17 00:00:00 2001 From: Katharina Eggensperger Date: Fri, 7 Mar 2014 11:03:18 +0100 Subject: [PATCH 083/139] FIX: check for runsolver --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 490ae1ed..3414d686 100644 --- a/setup.py +++ b/setup.py @@ -98,7 +98,7 @@ def _check_for_runsolver(self): return False for p in os.environ.get('PATH', '').split(os.pathsep): p = os.path.join(p, name) - if os.access(p, flags): + if os.access(p, flags) and not os.path.isdir(p): print "Copy runsolver from %s to runsolver/src/runsolver" % p shutil.copy(p, os.path.join(os.getcwd(), 'runsolver/src/runsolver')) return p From 44f2e1b0a4dfa85b6097b770c0073649ba703fd7 Mon Sep 17 00:00:00 2001 From: Katharina Eggensperger Date: Fri, 7 Mar 2014 11:18:45 +0100 Subject: [PATCH 084/139] FIX: create runsolver/src if runsolver is found --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 3414d686..fd13a030 100644 --- a/setup.py +++ b/setup.py @@ -99,6 +99,8 @@ def _check_for_runsolver(self): for p in os.environ.get('PATH', '').split(os.pathsep): p = os.path.join(p, name) if os.access(p, flags) and not os.path.isdir(p): + if not os.path.isdir(os.path.join(os.getcwd(), 'runsolver/src/')): + os.mkdir(os.path.join(os.getcwd(), 'runsolver/src/')) print "Copy runsolver from %s to runsolver/src/runsolver" % p shutil.copy(p, os.path.join(os.getcwd(), 'runsolver/src/runsolver')) return p From bc5cc4d5adaa032b9c76a054e03b7bfc7d0e0036 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Fri, 7 Mar 2014 11:41:20 +0100 Subject: [PATCH 085/139] FIX: Add check_dependencies --- .../tpe/random_hyperopt_august2013_mod.py | 43 ++++++++++++++++--- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/optimizers/tpe/random_hyperopt_august2013_mod.py b/optimizers/tpe/random_hyperopt_august2013_mod.py index 4344fcde..27eb9327 100644 --- a/optimizers/tpe/random_hyperopt_august2013_mod.py +++ b/optimizers/tpe/random_hyperopt_august2013_mod.py @@ -23,7 +23,7 @@ import os import sys -import HPOlib.wrapping_util as wrapping_util +import HPOlib.wrapping_util as wrappingUtil __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" @@ -32,8 +32,36 @@ logger = logging.getLogger("HPOlib.optimizers.tpe.randomtpe") version_info = ("# %76s #" % "https://github.com/hyperopt/hyperopt/tree/486aebec8a4170e4781d99bbd6cca09123b12717") -__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] -__contact__ = "automl.org" + + +# noinspection PyUnresolvedReferences +def check_dependencies(): + try: + import nose + except ImportError: + raise ImportError("Nose cannot be imported. Are you sure it's " + "installed?") + try: + import networkx + except ImportError: + raise ImportError("Networkx cannot be imported. Are you sure it's " + "installed?") + try: + import pymongo + from bson.objectid import ObjectId + except ImportError: + raise ImportError("Pymongo cannot be imported. Are you sure it's" + " installed?") + try: + import numpy + except ImportError: + raise ImportError("Numpy cannot be imported. Are you sure that it's" + " installed?") + try: + import scipy + except ImportError: + raise ImportError("Scipy cannot be imported. Are you sure that it's" + " installed?") def build_random_call(config, options, optimizer_dir): @@ -50,17 +78,18 @@ def build_random_call(config, options, optimizer_dir): return call +# noinspection PyUnusedLocal def restore(config, optimizer_dir, **kwargs): restore_file = os.path.join(optimizer_dir, 'state.pkl') if not os.path.exists(restore_file): logger.error("Oups, this should have been checked before") raise Exception("%s does not exist" % (restore_file,)) - return -1 fh = open(restore_file) state = cPickle.load(fh) fh.close() complete_runs = 0 + # noinspection PyProtectedMember tpe_trials = state['trials']._trials for trial in tpe_trials: # Assumes that all states no valid state is marked crashed @@ -70,12 +99,13 @@ def restore(config, optimizer_dir, **kwargs): return restored_runs +# noinspection PyUnusedLocal def main(config, options, experiment_dir, **kwargs): # config: Loaded .cfg file # options: Options containing seed, restore, # experiment_dir: Experiment directory/Benchmarkdirectory # **kwargs: Nothing so far - time_string = wrapping_util.get_time_string() + time_string = wrappingUtil.get_time_string() cmd = "" # Add path_to_optimizer to PYTHONPATH and to sys.path @@ -120,6 +150,5 @@ def main(config, options, experiment_dir, **kwargs): logger.info("%s" % version_info) logger.info("# A newer version might be available, but not yet built in. #") logger.info("################################################################################") - return cmd, optimizer_dir - return cmd, optimizer_dir \ No newline at end of file + return cmd, optimizer_dir From dcaaede05e43db735aa4267f276cc289cdb06eab Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Fri, 7 Mar 2014 13:08:03 +0100 Subject: [PATCH 086/139] FIX Setting PYTHONPATH in tpe and random --- optimizers/tpe/hyperopt_august2013_mod.py | 5 ++++- optimizers/tpe/random_hyperopt_august2013_mod.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/optimizers/tpe/hyperopt_august2013_mod.py b/optimizers/tpe/hyperopt_august2013_mod.py index 4c4a67c3..a5784701 100644 --- a/optimizers/tpe/hyperopt_august2013_mod.py +++ b/optimizers/tpe/hyperopt_august2013_mod.py @@ -110,7 +110,10 @@ def main(config, options, experiment_dir, **kwargs): # Add path_to_optimizer to PYTHONPATH and to sys.path # Only for HYPEROPT - os.environ['PYTHONPATH'] = config.get('TPE', 'path_to_optimizer') + os.pathsep + os.environ['PYTHONPATH'] + if not 'PYTHONPATH' in os.environ: + os.environ['PYTHONPATH'] = config.get('TPE', 'path_to_optimizer') + else: + os.environ['PYTHONPATH'] = config.get('TPE', 'path_to_optimizer') + os.pathsep + os.environ['PYTHONPATH'] sys.path.append(config.get('TPE', 'path_to_optimizer')) optimizer_str = os.path.splitext(os.path.basename(__file__))[0] diff --git a/optimizers/tpe/random_hyperopt_august2013_mod.py b/optimizers/tpe/random_hyperopt_august2013_mod.py index 27eb9327..b06e5864 100644 --- a/optimizers/tpe/random_hyperopt_august2013_mod.py +++ b/optimizers/tpe/random_hyperopt_august2013_mod.py @@ -109,7 +109,10 @@ def main(config, options, experiment_dir, **kwargs): cmd = "" # Add path_to_optimizer to PYTHONPATH and to sys.path - os.environ['PYTHONPATH'] = config.get('TPE', 'path_to_optimizer') + os.pathsep + os.environ['PYTHONPATH'] + if not 'PYTHONPATH' in os.environ: + os.environ['PYTHONPATH'] = config.get('TPE', 'path_to_optimizer') + else: + os.environ['PYTHONPATH'] = config.get('TPE', 'path_to_optimizer') + os.pathsep + os.environ['PYTHONPATH'] sys.path.append(config.get('TPE', 'path_to_optimizer')) optimizer_str = os.path.splitext(os.path.basename(__file__))[0] From f60e2a36dc6d67760cf8078a4c7168ebd5e6e531 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Fri, 7 Mar 2014 13:18:39 +0100 Subject: [PATCH 087/139] FIX directory where to save plots for doAllplots --- HPOlib/Plotting/doAllPlots.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/HPOlib/Plotting/doAllPlots.py b/HPOlib/Plotting/doAllPlots.py index 85c9cc7a..d48de251 100644 --- a/HPOlib/Plotting/doAllPlots.py +++ b/HPOlib/Plotting/doAllPlots.py @@ -146,6 +146,7 @@ def _statistics(pkl_list, name_list, save=""): def main(): + prog = "python doAllPlots.py WhatIsThis [WhatIsThis ]" description = "Tries to save as many plots as possible" @@ -157,11 +158,15 @@ def main(): parser.add_argument("-s", "--save", dest="save", default="", help="Where to save plots? (directory)") + parser.add_argument("-f", "--file", dest="file", + default="png", help="File ending") + args, unknown = parser.parse_known_args() sys.stdout.write("Found " + str(len(unknown)) + " arguments\n") - save_dir = args.save + save_dir = os.path.realpath(args.save) + log = args.log pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) @@ -181,7 +186,7 @@ def main(): if len(name_list) == 1 and name_list[0][1] == 1: # We have one exp and one pkl if save_dir is not "": - tmp_save = os.path.join(save_dir, "plotTrace_%s.png" % time_str) + tmp_save = os.path.join(save_dir, "plotTrace_%s.%s" % (time_str, args.file)) else: tmp_save = save_dir sys.stdout.write("plotTrace.py ... %s ..." % tmp_save) @@ -191,7 +196,7 @@ def main(): # Some plots only make sense, if there are many experiments # BoxWhisker if save_dir is not "": - tmp_save = os.path.join(save_dir, "BoxWhisker_%s.png" % time_str) + tmp_save = os.path.join(save_dir, "BoxWhisker_%s.%s" % (time_str, args.file)) else: tmp_save = save_dir sys.stdout.write("plotBoxWhisker.py ... %s ..." % tmp_save) @@ -208,7 +213,7 @@ def main(): # We can always plot this # OptimizerOverhead if save_dir is not "": - tmp_save = os.path.join(save_dir, "OptimizerOverhead_%s.png" % time_str) + tmp_save = os.path.join(save_dir, "OptimizerOverhead_%s.%s" % (time_str, args.file)) else: tmp_save = save_dir sys.stdout.write("plotOptimizerOverhead.py ... %s ..." % tmp_save) @@ -216,14 +221,14 @@ def main(): # Error Trace with Std if save_dir is not "": - tmp_save = os.path.join(save_dir, "TraceWithStd_perEval_%s.png" % time_str) + tmp_save = os.path.join(save_dir, "TraceWithStd_perEval_%s.%s" % (time_str, args.file)) else: tmp_save = save_dir sys.stdout.write("TraceWithStd_perEval.py ... %s ..." % tmp_save) _trace_with_std_per_eval(pkl_list=pkl_list, name_list=name_list, save=tmp_save, log=log) if save_dir is not "": - tmp_save = os.path.join(save_dir, "TraceWithStd_perTime_%s.png" % time_str) + tmp_save = os.path.join(save_dir, "TraceWithStd_perTime_%s.%s" % (time_str, args.file)) else: tmp_save = save_dir sys.stdout.write("TraceWithStd_perTime.py ... %s ..." % tmp_save) From 89e43f18d8492dad28cd4900f97df3588c6214c1 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Fri, 7 Mar 2014 14:10:38 +0100 Subject: [PATCH 088/139] REFACTOR: Remove code smells --- HPOlib/Plotting/plotBranin.py | 134 ++++++++++++++++------------------ 1 file changed, 62 insertions(+), 72 deletions(-) diff --git a/HPOlib/Plotting/plotBranin.py b/HPOlib/Plotting/plotBranin.py index 96e34577..cf8e79f9 100644 --- a/HPOlib/Plotting/plotBranin.py +++ b/HPOlib/Plotting/plotBranin.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + ## # wrapping: A program making it easy to use hyperparameter # optimization software. @@ -15,30 +17,25 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - -#!/usr/bin/env python +from argparse import ArgumentParser import cPickle import itertools -import optparse -import os import sys -from matplotlib.colors import LogNorm -import matplotlib.cm as cm -import matplotlib.gridspec as gridspec -from matplotlib.mlab import griddata -import matplotlib.pyplot as plt +import matplotlib.cm +import matplotlib.gridspec as gridSpec +import matplotlib.pyplot import numpy as np -sys.path.append("../benchmarks/branin") -import branin +import HPOlib.benchmark_functions +import HPOlib.Plotting.plot_util as plotUtil __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" -def plot_contour(trialList, nameList, save = "", title=""): +def plot_contour(trial_list, name_list, save="", title=""): # constraints: # -5 <= x <= 10, 0 <= y <= 15 # three global optima: (-pi, 12.275), (pi, 2.275), (9.42478, 2.475), where @@ -46,15 +43,14 @@ def plot_contour(trialList, nameList, save = "", title=""): markers = itertools.cycle(['o', 's', '^', 'x']) colors = itertools.cycle(['b', 'g', 'r', 'k']) - # linestyles = itertools.cycle(['-']) size = 5 # Get handles - ratio=5 - gs = gridspec.GridSpec(ratio,1) - fig = plt.figure(1, dpi=100) + ratio = 5 + gs = gridSpec.GridSpec(ratio, 1) + fig = matplotlib.pyplot.figure(1, dpi=100) fig.suptitle(title) - ax = plt.subplot(gs[0:ratio, :]) + ax = matplotlib.pyplot.subplot(gs[0:ratio, :]) ax.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5) xopt = [-np.pi, np.pi, 9.42478] yopt = [12.275, 2.275, 2.475] @@ -68,85 +64,79 @@ def plot_contour(trialList, nameList, save = "", title=""): for i in range(len(xi)): for j in range(len(yi)): #z[j, i] = np.power(np.e, branin.branin({"x":xi[i], "y":yi[j]})) - z[j, i] = branin.branin({"x":xi[i], "y":yi[j]}) - XI, YI = np.meshgrid(xi, yi) - cax = ax.contourf(XI, YI, z, 50, cmap=cm.gray) + z[j, i] = HPOlib.benchmark_functions.branin(x=xi[i], y=yi[j]) + xi, yi = np.meshgrid(xi, yi) + cax = ax.contourf(xi, yi, z, 50, cmap=matplotlib.cm.gray) fig.colorbar(cax) # Plot Optimums after all work is done - d = plt.scatter(xopt, yopt, marker="o", facecolor='w', edgecolor='w', s = 20*size, label="Optimum") + matplotlib.pyplot.scatter(xopt, yopt, marker="o", facecolor='w', edgecolor='w', s=20*size, label="Optimum") # Get values - for opt in range(len(nameList)): - print nameList[opt], "has", len(trialList[opt]['trials']), "samples" + for opt in range(len(name_list)): + print name_list[opt], "has", len(trial_list[opt]['trials']), "samples" m = markers.next() c = colors.next() - x = np.zeros(len(trialList[opt]["trials"])) - y = np.zeros(len(trialList[opt]["trials"])) + x = np.zeros(len(trial_list[opt]["trials"])) + y = np.zeros(len(trial_list[opt]["trials"])) for i in range(len(x)): - x[i] = trialList[opt]["trials"][i]["params"]["x"] - y[i] = trialList[opt]["trials"][i]["params"]["y"] - a = plt.scatter(x[0:10], y[0:10], marker=m, - s=size, facecolors=c, linewidth=0.1) # label=nameList[opt]) # + " first 10 points") - b = plt.scatter(x[10:-10], y[10:-10], marker=m, - linewidth=0.1, s=4*size, facecolors=c) # label="points in the middle") - c = plt.scatter(x[-10:-1], y[-10:-1], marker=m, - linewidth=0.1, s=6*size, facecolors=c, label=nameList[opt]) # label="last 10 points") + x[i] = float(trial_list[opt]["trials"][i]["params"]["-x"].strip("'")) + y[i] = float(trial_list[opt]["trials"][i]["params"]["-y"].strip("'")) + matplotlib.pyplot.scatter(x[0:10], y[0:10], marker=m, + s=size, facecolors=c, linewidth=0.1) + matplotlib.pyplot.scatter(x[10:-10], y[10:-10], marker=m, + linewidth=0.1, s=4*size, facecolors=c) + matplotlib.pyplot.scatter(x[-10:-1], y[-10:-1], marker=m, + linewidth=0.1, s=6*size, facecolors=c, label=name_list[opt][0]) - plt.xlim([-5, 10]) - plt.ylim([-0, 15]) - plt.xlabel("X") - plt.ylabel("Y") - if [trial["result"] for trial in trialList[opt]["trials"]]: - minimum = str(min([trial["result"] for trial in trialList[opt]["trials"]])) - else: - minimum = "NaN" + matplotlib.pyplot.xlim([-5, 10]) + matplotlib.pyplot.ylim([-0, 15]) + matplotlib.pyplot.xlabel("X") + matplotlib.pyplot.ylabel("Y") # Describe the plot - plt.title(title) - leg = plt.legend(loc="best", fancybox=True) + matplotlib.pyplot.title(title) + leg = matplotlib.pyplot.legend(loc="best", fancybox=True) leg.get_frame().set_alpha(0.5) if save != "": - plt.subplots_adjust(top=0.85) - plt.savefig(save, dpi=600, facecolor='w', edgecolor='w', - orientation='portrait', papertype=None, format=None, - transparent=False, bbox_inches="tight", pad_inches=0.1) + matplotlib.pyplot.subplots_adjust(top=0.85) + matplotlib.pyplot.savefig(save, dpi=600, facecolor='w', edgecolor='w', + orientation='portrait', papertype=None, format=None, + transparent=False, bbox_inches="tight", pad_inches=0.1) else: - plt.show() + matplotlib.pyplot.show() + def main(): - usage = "Usage: python plotBranin.py SMAC TPE <..> ..." + \ - "[-s ] [-t ]" - parser = optparse.OptionParser(usage) - parser.add_option("-s", "--save", dest = "save", default = "", \ - help = "Where to save plot instead of showing it?") - parser.add_option("-t", "--title", dest="title", \ - default="", help="Optional supertitle for plot") - (opts, args) = parser.parse_args() - - if len(args)%2 != 0: + prog = "python plotBranin.py whatIsThis [whatIsThis] ]" + description = "Plot a Trace with std for multiple experiments" + + parser = ArgumentParser(description=description, prog=prog) + + parser.add_argument("-s", "--save", dest="save", default="", + help="Where to save plot instead of showing it?") + parser.add_argument("-t", "--title", dest="title", default="", + help="Optional supertitle for plot") + + args, unknown = parser.parse_known_args() + + if len(unknown) % 2 != 0: print "Wrong number of arguments", len(args) - print usage + print prog sys.exit(1) - trialList = list() - nameList = list() - for i in range(len(args)): - if not ".pkl" in args[i]: - print "Adding", args[i] - nameList.append(args[i]) - continue - result_file = args[i] + pkl_list, name_list = plotUtil.get_pkl_and_name_list(unknown) + trial_list = list() + for i in range(len(name_list)): + result_file = pkl_list[i][0] fh = open(result_file, "r") trials = cPickle.load(fh) fh.close() - trialList.append(trials) + trial_list.append(trials) - assert len(nameList) == len(trialList), ("Optimizers: %s != %s" % (len(nameList), len(trialList))) - plot_contour(trialList=trialList, nameList=nameList, save=opts.save, title=opts.title) + plot_contour(trial_list=trial_list, name_list=name_list, save=args.save, title=args.title) if __name__ == "__main__": - main() - + main() \ No newline at end of file From ebd09eb33725306293b055c7d22546bb469568b0 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Fri, 7 Mar 2014 14:21:37 +0100 Subject: [PATCH 089/139] REMOVE plothar6, because this plot makes no sense --- HPOlib/Plotting/plothar6.py | 245 ------------------------------------ 1 file changed, 245 deletions(-) delete mode 100644 HPOlib/Plotting/plothar6.py diff --git a/HPOlib/Plotting/plothar6.py b/HPOlib/Plotting/plothar6.py deleted file mode 100644 index d5cf2cdd..00000000 --- a/HPOlib/Plotting/plothar6.py +++ /dev/null @@ -1,245 +0,0 @@ -## -# wrapping: A program making it easy to use hyperparameter -# optimization software. -# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -#!/usr/bin/env python - -import cPickle -import optparse -import os -import sys - -from matplotlib.colors import LogNorm -from matplotlib.mlab import griddata -import matplotlib.pyplot as plt -import numpy as np -import math - -sys.path.append("../") -import functions.har6 as har6 - -__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] -__contact__ = "automl.org" - - -def plot_contour(trials, save="", title=""): - # 6d Hartmann function - # constraints: - # 0 <= xi <= 1, i = 1..6 - # global optimum at (0.20169, 0.150011, 0.476874, 0.275332, 0.311652, 0.6573), - # where har6 = -3.32237 - - xopt = 0.20169 - yopt = 0.150011 - zopt = 0.476874 - aopt = 0.275332 - bopt = 0.311652 - copt = 0.6573 - optimum = -3.32237 - - x = np.zeros(len(trials["trials"])) - y = np.zeros(len(trials["trials"])) - z = np.zeros(len(trials["trials"])) - a = np.zeros(len(trials["trials"])) - b = np.zeros(len(trials["trials"])) - c = np.zeros(len(trials["trials"])) - result = np.zeros(len(trials["params"])) - for i in range(len(x)): - x[i] = trials["trials"][i]["params"]["x"] - y[i] = trials["trials"][i]["params"]["y"] - z[i] = trials["trials"][i]["params"]["z"] - a[i] = trials["trials"][i]["params"]["a"] - b[i] = trials["trials"][i]["params"]["b"] - c[i] = trials["trials"][i]["params"]["c"] - result[i] = trials['results'][i] - optimum - if math.isnan(x[i]) or math.isnan(y[i]) or math.isnan(result[i]): - print "#################sdfkdfs!" - maxRes = max(result) - minRes = min(result) - - f = plt.figure(1, figsize = (10, 10), dpi = 100) - - # Find minimum to generate title - if trials["results"] != []: - minimum = str(min(trials["results"])) - else: - minimum = "NaN" - f.suptitle(title + \ - " with Hartmann6d\nUsed " + str(len(x)) + " trials, " \ - + "best found value: " + minimum + " (-3.32237)", fontsize=16) - - plt.subplot(611) - size = 10 - aplt = plt.scatter(x[0:10], result[0:10], marker="^", \ - s=size, facecolors="none", edgecolors="k", label="first 10 points") - bplt = plt.scatter(x[10:-10], result[10:-10], marker="o", \ - s=4*size, facecolors="none", edgecolors="k", label="points in the middle") - cplt = plt.scatter(x[-10:-1], result[-10:-1], marker="*", \ - s=4*4*size, facecolors="none", edgecolors="k", label="last 10 points") - plt.plot([xopt, xopt], [0, maxRes], color="r") - plt.xlim([0, 1]) - plt.ylim([0, maxRes]) - plt.xlabel("X") - plt.ylabel("Result - Optimum") - #plt.yscale('log') - leg = plt.legend((aplt, bplt, cplt), (aplt.get_label(), bplt.get_label(), \ - cplt.get_label()), fancybox=True, loc="best") - leg.get_frame().set_alpha(0.5) - - plt.subplot(612) - aplt = plt.scatter(y[0:10], result[0:10], marker="^", \ - s=size, facecolors="none", edgecolors="k", label="first 10 points") - bplt = plt.scatter(y[10:-10], result[10:-10], marker="o", \ - s=4*size, facecolors="none", edgecolors="k", label="points in the middle") - cplt = plt.scatter(y[-10:-1], result[-10:-1], marker="*", \ - s=4*4*size, facecolors="none", edgecolors="k", label="last 10 points") - plt.plot([yopt, yopt], [0, maxRes], color="r") - plt.xlim([0, 1]) - plt.ylim([0, maxRes]) - plt.xlabel("Y") - plt.ylabel("Result - Optimum") - #plt.yscale('log') - leg = plt.legend((aplt, bplt, cplt), (aplt.get_label(), bplt.get_label(), \ - cplt.get_label()), fancybox=True, loc="best") - leg.get_frame().set_alpha(0.5) - - plt.subplot(613) - aplt = plt.scatter(z[0:10], result[0:10], marker="^", \ - s=size, facecolors="none", edgecolors="k", label="first 10 points") - bplt = plt.scatter(z[10:-10], result[10:-10], marker="o", \ - s=4*size, facecolors="none", edgecolors="k", label="points in the middle") - cplt = plt.scatter(z[-10:-1], result[-10:-1], marker="*", \ - s=4*4*size, facecolors="none", edgecolors="k", label="last 10 points") - plt.plot([zopt, zopt], [0, maxRes], color="r") - - plt.xlim([0, 1]) - plt.ylim([0, maxRes]) - plt.xlabel("Z") - plt.ylabel("Result - Optimum") - #plt.yscale('log') - leg = plt.legend((aplt, bplt, cplt), (aplt.get_label(), bplt.get_label(), \ - cplt.get_label()), fancybox=True, loc="best") - leg.get_frame().set_alpha(0.5) - - plt.subplot(614) - aplt = plt.scatter(a[0:10], result[0:10], marker="^", \ - s=size, facecolors="none", edgecolors="k", label="first 10 points") - bplt = plt.scatter(a[10:-10], result[10:-10], marker="o", \ - s=4*size, facecolors="none", edgecolors="k", label="points in the middle") - cplt = plt.scatter(a[-10:-1], result[-10:-1], marker="*", \ - s=4*4*size, facecolors="none", edgecolors="k", label="last 10 points") - plt.plot([zopt, zopt], [0, maxRes], color="r") - - plt.xlim([0, 1]) - plt.ylim([0, maxRes]) - plt.xlabel("A") - plt.ylabel("Result - Optimum") - #plt.yscale('log') - leg = plt.legend((aplt, bplt, cplt), (aplt.get_label(), bplt.get_label(), \ - cplt.get_label()), fancybox=True, loc="best") - leg.get_frame().set_alpha(0.5) - - - plt.subplot(615) - aplt = plt.scatter(b[0:10], result[0:10], marker="^", \ - s=size, facecolors="none", edgecolors="k", label="first 10 points") - bplt = plt.scatter(b[10:-10], result[10:-10], marker="o", \ - s=4*size, facecolors="none", edgecolors="k", label="points in the middle") - cplt = plt.scatter(b[-10:-1], result[-10:-1], marker="*", \ - s=4*4*size, facecolors="none", edgecolors="k", label="last 10 points") - plt.plot([zopt, zopt], [0, maxRes], color="r") - - plt.xlim([0, 1]) - plt.ylim([0, maxRes]) - plt.xlabel("B") - plt.ylabel("Result - Optimum") - #plt.yscale('log') - leg = plt.legend((aplt, bplt, cplt), (aplt.get_label(), bplt.get_label(), \ - cplt.get_label()), fancybox=True, loc="best") - leg.get_frame().set_alpha(0.5) - - - plt.subplot(616) - aplt = plt.scatter(c[0:10], result[0:10], marker="^", \ - s=size, facecolors="none", edgecolors="k", label="first 10 points") - bplt = plt.scatter(c[10:-10], result[10:-10], marker="o", \ - s=4*size, facecolors="none", edgecolors="k", label="points in the middle") - cplt = plt.scatter(c[-10:-1], result[-10:-1], marker="*", \ - s=4*4*size, facecolors="none", edgecolors="k", label="last 10 points") - plt.plot([zopt, zopt], [0, maxRes], color="r") - - plt.xlim([0, 1]) - plt.ylim([0, maxRes]) - plt.xlabel("C") - plt.ylabel("Result - Optimum") - #plt.yscale('log') - leg = plt.legend((aplt, bplt, cplt), (aplt.get_label(), bplt.get_label(), \ - cplt.get_label()), fancybox=True, loc="best") - leg.get_frame().set_alpha(0.5) - - plt.tight_layout() - plt.subplots_adjust(top=0.85) - if save != "": - plt.savefig(save, dpi=100, facecolor='w', edgecolor='w', - orientation='portrait', papertype=None, format=None, - transparent=False, bbox_inches="tight", pad_inches=0.1) - else: - plt.show() - f.clear() - -def main(): - parser = optparse.OptionParser() - parser.add_option("-s", "--save", dest = "save", default = "", \ - help = "Where to save plot instead of showing it?") - (opts, args) = parser.parse_args() - - if opts.save != "": - sys.stdout.write("Creating %s plots\n" % (len(args),) ) - for idx,i in enumerate(args): - result_file = i - print i - if os.path.exists(result_file) != True: - sys.stderr.write("No valid file: %s\n" % (result_file,) ) - fh = open(result_file, "r") - trials = cPickle.load(fh) - fh.close() - idxx = result_file.find(trials["experiment_name"]) - title = result_file[idxx:-(len(trials["experiment_name"])+5)] - plot_contour(trials, save=opts.save + title, title=title) - print len(trials['results']) - else: - result_file = args[0] - fh = open(result_file, "r") - trials = cPickle.load(fh) - fh.close() - - """ - for i in range(10): - print "x:" + str(trials["params"][i]["x"]) + \ - " y:" + str(trials["params"][i]["y"]) + \ - " z:" + str(trials["params"][i]["z"]) + \ - " a:" + str(trials["params"][i]["a"]) + \ - " b:" + str(trials["params"][i]["b"]) + \ - " c:" + str(trials["params"][i]["c"]) + \ - " res:" + str(trials["results"][i]) - """ - idxx = result_file.find(trials["experiment_name"]) - title = result_file[idxx:-(len(trials["experiment_name"])+5)] - plot_contour(trials, title=title) - -if __name__ == "__main__": - main() From f47ac4bb3ffe7f70b795d50c0828ca7e109d2879 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 7 Mar 2014 15:16:05 +0100 Subject: [PATCH 090/139] ENHANCEMENT: #12 total_time_limit is now enforced --- HPOlib/wrapping.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 98e90cee..c888b1ce 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -21,6 +21,8 @@ import logging import os from Queue import Queue, Empty +import signal +import shlex import subprocess import sys from threading import Thread @@ -267,15 +269,22 @@ def main(): logger.info(cmd) output_file = os.path.join(optimizer_dir_in_experiment, optimizer + ".out") fh = open(output_file, "a") - - proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + cmd = shlex.split(cmd) + print cmd + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, preexec_fn=os.setsid) logger.info("-----------------------RUNNING----------------------------------") # http://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python + # How often is the experiment pickle supposed to be opened? last_output = time.time() + optimizer_end_time = time.time() + config.getint("HPOLIB", "total_time_limit") printed_start_configuration = list() printed_end_configuration = list() current_best = -1 + sent_SIGTERM = False + sent_SIGKILL = False + # After the evaluation finished, we scan the experiment pickle twice + # to print everything! minimal_runs_to_go = 2 def enqueue_output(out, queue): @@ -291,7 +300,7 @@ def enqueue_output(out, queue): stdout_thread.daemon = True stderr_thread.start() stdout_thread.start() - logger.info('Optimizer runs with PID (+1): %d', proc.pid) + logger.info('Optimizer runs with PID: %d', proc.pid) while minimal_runs_to_go > 0: # Think of this as a do-while loop... try: @@ -318,6 +327,14 @@ def enqueue_output(out, queue): except Empty: pass + if time.time() > optimizer_end_time and not sent_SIGTERM: + os.killpg(proc.pid, signal.SIGTERM) + sent_SIGTERM = True + + if time.time() > optimizer_end_time + 200 and not sent_SIGKILL: + os.killpg(proc.pid, signal.SIGKILL) + sent_SIGKILL + fh.flush() # necessary, otherwise HPOlib-run takes 100% of one processor time.sleep(0.1) From d9c50927ae88c85edb5350f15fa4355903fe5c5c Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 7 Mar 2014 15:23:41 +0100 Subject: [PATCH 091/139] REFACTOR: adapted optimizers for #12 TPE now changes the working directory itself TPE now doesn't rely on the config key algorithm any more Spearmint now calls HPOlib.cv instead of cv for the experiments REMOVED an update of the python path in wrapping.py --- HPOlib/config_parser/generalDefault.cfg | 1 - HPOlib/wrapping.py | 6 +----- benchmarks/branin/spearmint_april2013_mod/config.pb | 2 +- benchmarks/har6/spearmint_april2013_mod/config.pb | 2 +- optimizers/tpe/hyperopt_august2013_mod.py | 7 +++---- optimizers/tpe/random_hyperopt_august2013_mod.py | 12 +++++------- optimizers/tpe/tpecall.py | 12 +++++++----- 7 files changed, 18 insertions(+), 24 deletions(-) diff --git a/HPOlib/config_parser/generalDefault.cfg b/HPOlib/config_parser/generalDefault.cfg index f47bb281..f23a7758 100644 --- a/HPOlib/config_parser/generalDefault.cfg +++ b/HPOlib/config_parser/generalDefault.cfg @@ -1,6 +1,5 @@ [HPOLIB] #Will be used for wrapping.py and SMAC -algorithm = cv.py run_instance = runsolver_wrapper.py use_own_time_measurement = True numberCV = 1 diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index c888b1ce..13f68e61 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -173,10 +173,6 @@ def main(): # Try adding runsolver to path os.putenv('PATH', os.environ['PATH'] + ":" + wrapping_dir + "/../runsolver/src/") - # TODO: We also don't need this - # build call - cmd = "export PYTHONPATH=$PYTHONPATH:" + wrapping_dir + "\n" - # Load optimizer try: optimizer_dir = os.path.dirname(os.path.realpath(optimizer_version)) @@ -189,7 +185,7 @@ def main(): optimizer_call, optimizer_dir_in_experiment = optimizer_module.main(config=config, options=args, experiment_dir=experiment_dir) - cmd += optimizer_call + cmd = optimizer_call with open(os.path.join(optimizer_dir_in_experiment, "config.cfg"), "w") as f: config.set("HPOLIB", "is_not_original_config_file", "True") diff --git a/benchmarks/branin/spearmint_april2013_mod/config.pb b/benchmarks/branin/spearmint_april2013_mod/config.pb index 70137ce6..ddf59faa 100644 --- a/benchmarks/branin/spearmint_april2013_mod/config.pb +++ b/benchmarks/branin/spearmint_april2013_mod/config.pb @@ -1,5 +1,5 @@ language: PYTHON -name: "cv" +name: "HPOlib.cv" variable { name: "x" diff --git a/benchmarks/har6/spearmint_april2013_mod/config.pb b/benchmarks/har6/spearmint_april2013_mod/config.pb index d051740a..4d4ed4e0 100644 --- a/benchmarks/har6/spearmint_april2013_mod/config.pb +++ b/benchmarks/har6/spearmint_april2013_mod/config.pb @@ -1,5 +1,5 @@ language: PYTHON -name: "cv" +name: "HPOlib.cv" variable { name: "x" diff --git a/optimizers/tpe/hyperopt_august2013_mod.py b/optimizers/tpe/hyperopt_august2013_mod.py index 4c4a67c3..3434ee01 100644 --- a/optimizers/tpe/hyperopt_august2013_mod.py +++ b/optimizers/tpe/hyperopt_august2013_mod.py @@ -61,13 +61,12 @@ def check_dependencies(): def build_tpe_call(config, options, optimizer_dir): # For TPE we have to cd to the exp_dir - call = "cd " + optimizer_dir + "\n" + \ - "python " + os.path.dirname(os.path.realpath(__file__)) + \ + call = "python " + os.path.dirname(os.path.realpath(__file__)) + \ "/tpecall.py" call = ' '.join([call, '-p', config.get('TPE', 'space'), - "-a", config.get('HPOLIB', 'algorithm'), "-m", config.get('TPE', 'numberEvals'), - "-s", str(options.seed)]) + "-s", str(options.seed), + "--cwd", optimizer_dir]) if options.restore: call = ' '.join([call, '-r']) return call diff --git a/optimizers/tpe/random_hyperopt_august2013_mod.py b/optimizers/tpe/random_hyperopt_august2013_mod.py index 4344fcde..df851d3c 100644 --- a/optimizers/tpe/random_hyperopt_august2013_mod.py +++ b/optimizers/tpe/random_hyperopt_august2013_mod.py @@ -37,14 +37,12 @@ def build_random_call(config, options, optimizer_dir): - #For TPE (and Random Search) we have to cd to the exp_dir - call = 'cd ' + optimizer_dir + '\n' + \ - 'python ' + os.path.dirname(os.path.realpath(__file__)) + \ - '/tpecall.py' + call = "python " + os.path.dirname(os.path.realpath(__file__)) + \ + "/tpecall.py" call = ' '.join([call, '-p', config.get('TPE', 'space'), - '-a', config.get('HPOLIB', 'algorithm'), - '-m', config.get('HPOLIB', 'numberOfJobs'), - '-s', str(options.seed), '--random']) + "-m", config.get('TPE', 'numberEvals'), + "-s", str(options.seed), + "--cwd", optimizer_dir], "--random") if options.restore: call = ' '.join([call, '-r']) return call diff --git a/optimizers/tpe/tpecall.py b/optimizers/tpe/tpecall.py index 08d187d7..0bcaf632 100644 --- a/optimizers/tpe/tpecall.py +++ b/optimizers/tpe/tpecall.py @@ -27,6 +27,7 @@ import sys import hyperopt +import HPOlib.cv as cv logger = logging.getLogger("HPOlib.optimizers.tpe.tpecall") @@ -66,8 +67,6 @@ def main(): parser.add_argument("-p", "--space", dest="spaceFile", help="Where is the space.py located?") - parser.add_argument("-a", "--algoExec", - dest="algoExec", help="Which function to load located?") parser.add_argument("-m", "--maxEvals", dest="maxEvals", help="How many evaluations?") parser.add_argument("-s", "--seed", default="1", @@ -77,15 +76,19 @@ def main(): "the current working directory") parser.add_argument("--random", default=False, action="store_true", dest="random", help="Use a random search") + parser.add_argument("--cwd", help="Change the working directory before " + "optimizing.") args, unknown = parser.parse_known_args() + if args.cwd: + os.chdir(args.cwd) + if not os.path.exists(args.spaceFile): logger.critical("Search space not found: %s" % args.spaceFile) sys.exit(1) # First remove ".py" - algo, ext = os.path.splitext(os.path.basename(args.algoExec)) space, ext = os.path.splitext(os.path.basename(args.spaceFile)) # Then load dict searchSpace and out function cv.py @@ -94,8 +97,7 @@ def main(): module = import_module(space) search_space = module.space - fn = import_module(algo) - fn = fn.main # doForTPE + fn = cv.main # doForTPE if args.random: # We use a random search From b28971c1c9e7567fd2a8f51104eb5c0403aa0c60 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 7 Mar 2014 15:35:49 +0100 Subject: [PATCH 092/139] FIX: syntax error for random search --- optimizers/tpe/random_hyperopt_august2013_mod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimizers/tpe/random_hyperopt_august2013_mod.py b/optimizers/tpe/random_hyperopt_august2013_mod.py index d29616d6..6559763c 100644 --- a/optimizers/tpe/random_hyperopt_august2013_mod.py +++ b/optimizers/tpe/random_hyperopt_august2013_mod.py @@ -70,7 +70,7 @@ def build_random_call(config, options, optimizer_dir): call = ' '.join([call, '-p', config.get('TPE', 'space'), "-m", config.get('TPE', 'numberEvals'), "-s", str(options.seed), - "--cwd", optimizer_dir], "--random") + "--cwd", optimizer_dir, "--random"]) if options.restore: call = ' '.join([call, '-r']) return call From 29347af13ab55e0554760bc0833ed6af1aa576ba Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 7 Mar 2014 15:43:25 +0100 Subject: [PATCH 093/139] FIX: updated spearmint package checksum in setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fd13a030..92216831 100644 --- a/setup.py +++ b/setup.py @@ -199,7 +199,7 @@ def run(self): optimizer_tar_name="spearmint_april2013_mod_src.tar.gz", url="http://www.automl.org/spearmint_april2013" + "_mod_src.tar.gz", - md5='aa3eb7961637f9bf2a8cb9b14c4b8791') + md5='340fc0da97a30454d633ce9781b05369') # Do whatever you want to do # TODO: Normally one wants to call run(self), but this runs distutils and ignores install_requirements for unknown reasons From 31650df62afa87a2a640a0d87c11641e3c3a6a31 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Mon, 10 Mar 2014 10:46:16 +0100 Subject: [PATCH 094/139] FIX: broken statement in wrapping.py --- HPOlib/wrapping.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 13f68e61..549db39f 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -329,7 +329,7 @@ def enqueue_output(out, queue): if time.time() > optimizer_end_time + 200 and not sent_SIGKILL: os.killpg(proc.pid, signal.SIGKILL) - sent_SIGKILL + sent_SIGKILL = True fh.flush() # necessary, otherwise HPOlib-run takes 100% of one processor @@ -398,4 +398,4 @@ def enqueue_output(out, queue): return ret if __name__ == "__main__": - main() \ No newline at end of file + main() From 40bd4333d7ce152c85d78e464f01bdc2e77867ca Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 10 Mar 2014 10:53:00 +0100 Subject: [PATCH 095/139] REFACTOR: result.sh updated --- HPOlib/Plotting/results.sh | 44 +++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/HPOlib/Plotting/results.sh b/HPOlib/Plotting/results.sh index 4d6db764..fea13d2a 100644 --- a/HPOlib/Plotting/results.sh +++ b/HPOlib/Plotting/results.sh @@ -1,4 +1,4 @@ -#!/bin/sh + directory=`ls | grep "^smac_2_06_01-dev\>"` if [ -a "${directory}" ] @@ -86,22 +86,46 @@ then directory=`ls | grep "^hyperopt_august2013_mod_${i}_"` if [ -a "${directory}/hyperopt_august2013_mod.out" ] then - num=`cat ${directory}/hyperopt_august2013_mod.out | grep "Result:" | wc -l` + num=`cat ${directory}/hyperopt_august2013_mod.out | grep " -----------------------RUNNING RUNSOLVER" | wc -l` - per=`cat ${directory}/hyperopt_august2013_mod.out | grep "Result:" | sort -r | tail -1` - per=`echo $per | cut -d' ' -f2` + per=`cat ${directory}/hyperopt_august2013_mod.out | grep "Result for ParamILS:" | sort -r | tail -1` + per=`echo $per | cut -d' ' -f9` + per=`echo $per | sed 's/,//'` numC=`ls ${directory}/ | grep 'instance.out$' | wc -l` numC=$(($numC - 1)) test_error=999 test_file=`ls -1 "$directory" | grep "_test_run.out"` - if [ -f "$directory/$test_file" ] - then - test_error=`cat "$directory/$test_file" | grep "Result for ParamILS: SAT" | tail -1` - test_error=`echo "$test_error" | cut -d' ' -f7| cut -d',' -f1` - fi - printf "%5s | %4s(%5s) | %10f | %10f\n" "$i" "$num" "$numC" "$per" "$test_error" + printf "%5s | %4s(%5s) | %10f \n" "$i" "$num" "$numC" "$per" + fi + done +fi + + +directory=`ls | grep "^random_hyperopt_august2013_mod\>"` +if [ -a "${directory}" ] +then + echo "Looking for RandomTPE:" + + printf "%5s | %4s(%5s) | %10s | %10s\n" "Seed" "#run" "crash" "Performance" "Test-Perf" + for i in `seq 1000 1000 10000` + do + directory=`ls | grep "^random_hyperopt_august2013_mod_${i}_"` + if [ -a "${directory}/random_hyperopt_august2013_mod.out" ] + then + num=`cat ${directory}/random_hyperopt_august2013_mod.out | grep " -----------------------RUNNING RUNSOLVER" | wc -l` + + per=`cat ${directory}/random_hyperopt_august2013_mod.out | grep "Result for ParamILS:" | sort -r | tail -1` + per=`echo $per | cut -d' ' -f9` + per=`echo $per | sed 's/,//'` + + numC=`ls ${directory}/ | grep 'instance.out$' | wc -l` + numC=$(($numC - 1)) + + test_error=999 + test_file=`ls -1 "$directory" | grep "_test_run.out"` + printf "%5s | %4s(%5s) | %10f \n" "$i" "$num" "$numC" "$per" fi done fi From 1a8dc2333b0eafa2bce116f0eda5bbf74393d0bf Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 10 Mar 2014 10:54:14 +0100 Subject: [PATCH 096/139] REFACTOR: Minor changes --- HPOlib/Plotting/plotTraceWithStd_perEval.py | 2 +- HPOlib/Plotting/plotTraceWithStd_perTime.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HPOlib/Plotting/plotTraceWithStd_perEval.py b/HPOlib/Plotting/plotTraceWithStd_perEval.py index 937a3ef0..41b3d15f 100644 --- a/HPOlib/Plotting/plotTraceWithStd_perEval.py +++ b/HPOlib/Plotting/plotTraceWithStd_perEval.py @@ -105,7 +105,7 @@ def plot_optimization_trace(trial_list, name_list, optimum=0, title="", ax1.set_ylim([min_val-0.1*abs((max_val-min_val)), max_val+0.1*abs((max_val-min_val))]) else: ax1.set_ylim([y_min, y_max]) - ax1.set_xlim([0, 100]) # maxTrials]) + ax1.set_xlim([0, max_trials + 1]) tight_layout() subplots_adjust(top=0.85) diff --git a/HPOlib/Plotting/plotTraceWithStd_perTime.py b/HPOlib/Plotting/plotTraceWithStd_perTime.py index 6d2de297..78f1f95a 100644 --- a/HPOlib/Plotting/plotTraceWithStd_perTime.py +++ b/HPOlib/Plotting/plotTraceWithStd_perTime.py @@ -100,7 +100,7 @@ def plot_optimization_trace(trial_list, name_list, times_list, optimum=0, title= # Descript and label the stuff leg = ax1.legend(loc='best', fancybox=True) leg.get_frame().set_alpha(0.5) - ax1.set_xlabel("duration") + ax1.set_xlabel("Duration [sec] ") if y_max == y_min: # Set axes limit From ada59dbcdd3a8f2de33cdea40d768c1b15e6f344 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Mon, 10 Mar 2014 15:16:02 +0100 Subject: [PATCH 097/139] FIX: wrapping.py crashed with SMAC's intensify --- HPOlib/Experiment.py | 2 +- HPOlib/wrapping.py | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/HPOlib/Experiment.py b/HPOlib/Experiment.py index d6085c73..1dbdc1b4 100644 --- a/HPOlib/Experiment.py +++ b/HPOlib/Experiment.py @@ -177,7 +177,7 @@ def get_arg_best(self): best_value = sys.maxint for i, trial in enumerate(self.trials): tmp_res = np.NaN - if trial['result'] == trial['result']: + if np.isfinite(trial['result']): tmp_res = trial['result'] elif np.isfinite(trial['instance_results']).any(): tmp_res = scipy.nanmean(trial['instance_results']) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 13f68e61..75869737 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -340,22 +340,26 @@ def enqueue_output(out, queue): optimizer) for i in range(len(printed_end_configuration), len(trials.instance_order)): + configuration = trials.instance_order[i][0] + fold = trials.instance_order[i][1] if i + 1 > len(printed_start_configuration): logger.info("Starting configuration %5d, fold %2d", - trials.instance_order[i][0], - trials.instance_order[i][1]) + configuration, fold) printed_start_configuration.append(i) - if np.isfinite(trials.trials[i]["instance_results"][ - trials.instance_order[i][1]]): - last_result = trials.trials[i]["instance_results"][ - trials.instance_order[i][1]] + if np.isfinite(trials.trials[configuration] + ["instance_results"][fold]): + last_result = trials.trials[configuration] \ + ["instance_results"][fold] tmp_current_best = trials.get_arg_best() if tmp_current_best <= i: current_best = tmp_current_best + res = trials.trials[current_best]["result"] if \ + np.isfinite(trials.trials[current_best]["result"]) \ + else np.nanmean(trials.trials[current_best][ + "instance_results"]) logger.info("Result %10f, current best %10f", - last_result, - trials.trials[current_best]["result"]) + last_result, res) printed_end_configuration.append(i) del trials From 1891acd9fcd8dcaa2a55d0e88dcc692ce24d15fc Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Mon, 10 Mar 2014 15:22:02 +0100 Subject: [PATCH 098/139] ENHANCEMENT: #33 --- HPOlib/config_parser/generalDefault.cfg | 1 + HPOlib/runsolver_wrapper.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/HPOlib/config_parser/generalDefault.cfg b/HPOlib/config_parser/generalDefault.cfg index f23a7758..74c8d210 100644 --- a/HPOlib/config_parser/generalDefault.cfg +++ b/HPOlib/config_parser/generalDefault.cfg @@ -4,6 +4,7 @@ run_instance = runsolver_wrapper.py use_own_time_measurement = True numberCV = 1 max_crash_per_cv = 3 +remove_target_algorithm_output = True numberOfConcurrentJobs = 1 diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 51084589..423297f2 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -262,8 +262,8 @@ def parse_output_files(cfg, run_instance_output, runsolver_output_file): "result_on_terminate"), additional_data) elif error is None: - # Remove the run_instance_output only if there is a valid result - os.remove(run_instance_output) + if cfg.getboolean("HPOLIB", "remove_target_algorithm_output"): + os.remove(run_instance_output) os.remove(runsolver_output_file) rval = (cpu_time, wallclock_time, "SAT", float(result_array[6].strip(",")), cfg.get("HPOLIB", "function")) From b6f2f8bf40a8bff594c52a80faf4254e76eed6d7 Mon Sep 17 00:00:00 2001 From: Katharina Eggensperger Date: Mon, 10 Mar 2014 17:14:11 +0100 Subject: [PATCH 099/139] FIX: spearmint crashes with scipy==0.13.3 --- INSTALL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index da2fcbaf..212f6492 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -18,7 +18,7 @@ source virtualHPOlib/bin/activate ```bash easy_install -U distribute pip install numpy -pip install scipy +pip install scipy==0.13.2 pip install matplotlib ``` This may take some time. Afterwards you can verify having those libs installed with: @@ -33,7 +33,7 @@ pip freeze numpy==1.8.0 pyparsing==2.0.1 python-dateutil==2.2 - scipy==0.13.3 + scipy==0.13.2 six==1.5.2 tornado==3.2 wsgiref==0.1.2 From 8f6b0ba12a3ec37d0445e25c03c3e8073834a132 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 11 Mar 2014 09:37:22 +0100 Subject: [PATCH 100/139] FIX: better feedback when algorithm crashes So far user just receives an HPOlib error, when the algorithm output does not contain "Result for ...". HPOlib now prints a message and continues. --- HPOlib/runsolver_wrapper.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 51084589..fdaca970 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -141,6 +141,9 @@ def read_run_instance_output(run_instance_output): result_array = [value.strip(",") for value in result_array] break + if not result_string: + result_string = "".join(run_instance_content[-3:]) + return result_array, result_string @@ -234,7 +237,13 @@ def parse_output_files(cfg, run_instance_output, runsolver_output_file): cpu_time, measured_wallclock_time, error = read_runsolver_output( runsolver_output_file) result_array, result_string = read_run_instance_output(run_instance_output) - instance_wallclock_time = float(result_array[4]) + if not result_array: + logger.critical("We could not find anything matching our regexp. Last lines of your output:\n%s" + % result_string) + instance_wallclock_time = cfg.getfloat("HPOLIB", "runsolver_time_limit") + result_array = [None]*7 + else: + instance_wallclock_time = float(result_array[4]) if cfg.getboolean("HPOLIB", "use_own_time_measurement") is True: wallclock_time = measured_wallclock_time @@ -312,7 +321,7 @@ def main(): # Side-effect: removes all additional information like log and applies # transformations to the parameters params = get_parameters() - param_string = " ".join([key + " " + params[key] for key in params]) + param_string = " ".join([key + " " + str(params[key]) for key in params]) time_string = wrappingUtil.get_time_string() run_instance_output = os.path.join(os.getcwd(), @@ -328,7 +337,7 @@ def main(): trial_index = get_trial_index(experiment, fold, params) experiment.set_one_fold_running(trial_index, fold) del experiment # release Experiment lock - + logger.debug("Calling: %s" % cmd) process = subprocess.Popen(cmd, stdout=fh, stderr=fh, shell=True, executable="/bin/bash") @@ -347,6 +356,7 @@ def main(): result = cfg.getfloat("HPOLIB", "result_on_terminate") experiment.set_one_fold_crashed(trial_index, fold, result, wallclock_time) + status = "SAT" else: # TODO: We need a global stopping mechanism pass From 54fe1a5c6de308222e558a6aa35fa25ff0381e8e Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Tue, 11 Mar 2014 10:42:06 +0100 Subject: [PATCH 101/139] REFACTOR: issue #32 --- HPOlib/config_parser/generalDefault.cfg | 4 +-- HPOlib/config_parser/parse.py | 6 ++-- HPOlib/cv.py | 2 +- HPOlib/runsolver_wrapper.py | 8 +++-- HPOlib/wrapping.py | 2 +- benchmarks/branin/config.cfg | 2 +- benchmarks/har6/config.cfg | 2 +- optimizers/smac/smac_2_06_01-dev.py | 29 ++++++++++--------- optimizers/smac/smac_2_06_01-devDefault.cfg | 29 ++++++++++--------- optimizers/smac/smac_2_06_01-dev_parser.py | 22 +++++++------- .../spearmint/spearmint_april2013_mod.py | 9 ++---- .../spearmint_april2013_modDefault.cfg | 2 +- .../spearmint_april2013_mod_parser.py | 2 +- optimizers/tpe/hyperopt_august2013_mod.py | 4 +-- .../tpe/hyperopt_august2013_modDefault.cfg | 2 +- .../tpe/hyperopt_august2013_mod_parser.py | 4 +-- .../tpe/random_hyperopt_august2013_mod.py | 4 +-- tests/unittests/branin_test/config.cfg | 4 +-- tests/unittests/dummy_config.cfg | 2 +- 19 files changed, 70 insertions(+), 69 deletions(-) diff --git a/HPOlib/config_parser/generalDefault.cfg b/HPOlib/config_parser/generalDefault.cfg index 74c8d210..173a8670 100644 --- a/HPOlib/config_parser/generalDefault.cfg +++ b/HPOlib/config_parser/generalDefault.cfg @@ -2,11 +2,11 @@ #Will be used for wrapping.py and SMAC run_instance = runsolver_wrapper.py use_own_time_measurement = True -numberCV = 1 +number_cv_folds = 1 max_crash_per_cv = 3 remove_target_algorithm_output = True -numberOfConcurrentJobs = 1 +number_of_concurrent_jobs = 1 #The mem-limit must be expressed in mega-bytes #The cpu-limit must be expressed in seconds (CPU time) diff --git a/HPOlib/config_parser/parse.py b/HPOlib/config_parser/parse.py index a1649167..588542f7 100644 --- a/HPOlib/config_parser/parse.py +++ b/HPOlib/config_parser/parse.py @@ -44,9 +44,9 @@ def parse_config(config_files, allow_no_value=True, optimizer_version="", # -------------------------------------------------------------------------- # Check critical values # -------------------------------------------------------------------------- - if not config.has_option('HPOLIB', 'numberOfJobs') or \ - config.get('HPOLIB', 'numberOfJobs') == '': - raise Exception('numberOfJobs not specified in .cfg') + if not config.has_option('HPOLIB', 'number_of_jobs') or \ + config.get('HPOLIB', 'number_of_jobs') == '': + raise Exception('number_of_jobs not specified in .cfg') if not config.has_option('HPOLIB', 'result_on_terminate') or \ config.get('HPOLIB', 'result_on_terminate') == '': raise Exception('No result_on_terminate specified in .cfg') diff --git a/HPOlib/cv.py b/HPOlib/cv.py index d7a0f7f3..4b3bc07f 100644 --- a/HPOlib/cv.py +++ b/HPOlib/cv.py @@ -218,7 +218,7 @@ def main(*args, **kwargs): cfg = load_experiment_config_file() # Load number of folds - folds = cfg.getint('HPOLIB', 'numberCV') + folds = cfg.getint('HPOLIB', 'number_cv_folds') params = flatten_parameter_dict(params) diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 67d9af2d..16ed5e92 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -141,7 +141,10 @@ def read_run_instance_output(run_instance_output): result_array = [value.strip(",") for value in result_array] break - if not result_string: + # If we do not find a result string, return the last three lines of the + # run_instance output + # TODO: there must be some better way to tell the user what happend + if not result_string and len(run_instance_content) >= 3: result_string = "".join(run_instance_content[-3:]) return result_array, result_string @@ -214,7 +217,8 @@ def make_command(cfg, fold, param_string, run_instance_output): cpu_limit = cfg.getint('HPOLIB', 'cpu_limit') fn_filename = get_function_filename(cfg) python_cmd = cfg.get("HPOLIB", "leading_algo_info") + " python " + fn_filename - python_cmd += " --fold %d --folds %d --params %s" % (fold, cfg.getint("HPOLIB", "numberCV"), param_string) + python_cmd += " --fold %d --folds %d --params %s" % (fold, cfg.getint( + "HPOLIB", "number_cv_folds"), param_string) # Do not write the actual task in quotes because runsolver will not work # then; also we need use-pty and timestamp so that the "solver" output # is flushed to the output directory diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index e3c5d479..f5d6b6ac 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -200,7 +200,7 @@ def main(): os.remove(os.path.join(optimizer_dir_in_experiment, optimizer + ".pkl.lock")) except OSError: pass - folds = config.getint('HPOLIB', 'numberCV') + folds = config.getint('HPOLIB', 'number_cv_folds') trials = Experiment.Experiment(optimizer_dir_in_experiment, optimizer, folds=folds, max_wallclock_time=config.get('HPOLIB', 'cpu_limit'), diff --git a/benchmarks/branin/config.cfg b/benchmarks/branin/config.cfg index 1aabf8e4..d7133a8b 100644 --- a/benchmarks/branin/config.cfg +++ b/benchmarks/branin/config.cfg @@ -3,5 +3,5 @@ p = smac_2_06_01-dev/params.txt [HPOLIB] function = branin.py -numberOfJobs = 200 +number_of_jobs = 200 result_on_terminate = 1000 diff --git a/benchmarks/har6/config.cfg b/benchmarks/har6/config.cfg index f2643e39..3d632034 100644 --- a/benchmarks/har6/config.cfg +++ b/benchmarks/har6/config.cfg @@ -3,7 +3,7 @@ p = smac_2_06_01-dev/params.txt [HPOLIB] function = har6.py -numberOfJobs = 200 +number_of_jobs = 200 result_on_terminate = 1 # Overwrite diff --git a/optimizers/smac/smac_2_06_01-dev.py b/optimizers/smac/smac_2_06_01-dev.py index b07693e9..2b038d55 100644 --- a/optimizers/smac/smac_2_06_01-dev.py +++ b/optimizers/smac/smac_2_06_01-dev.py @@ -82,27 +82,28 @@ def build_smac_call(config, options, optimizer_dir): call = config.get('SMAC', 'path_to_optimizer') + "/smac" call = " ".join([call, '--numRun', str(options.seed), '--scenario-file', os.path.join(optimizer_dir, 'scenario.txt'), - '--cutoffTime', config.get('SMAC', 'cutoffTime'), + '--cutoffTime', config.get('SMAC', 'cutoff_time'), # The instance file does interfere with state restoration, it will only # be loaded if no state is restored (look further down in the code # '--instanceFile', config.get('SMAC', 'instanceFile'), - '--intraInstanceObj', config.get('SMAC', 'intraInstanceObj'), - '--runObj', config.get('SMAC', 'runObj'), + '--intraInstanceObj', config.get('SMAC', 'intra_instance_obj'), + '--runObj', config.get('SMAC', 'run_obj'), # '--testInstanceFile', config.get('SMAC', 'testInstanceFile'), - '--algoExec', '"python', os.path.join(algo_exec_dir, config.get('SMAC', 'algoExec')) + '"', + '--algoExec', '"python', os.path.join(algo_exec_dir, + config.get('SMAC', 'algo_exec')) + '"', '--execDir', optimizer_dir, '-p', config.get('SMAC', 'p'), # The experiment dir MUST not be specified when restarting, it is set # further down in the code # '--experimentDir', optimizer_dir, - '--numIterations', config.get('SMAC', 'numIterations'), - '--totalNumRunsLimit', config.get('SMAC', 'totalNumRunsLimit'), + '--numIterations', config.get('SMAC', 'num_iterations'), + '--totalNumRunsLimit', config.get('SMAC', 'total_num_runs_limit'), '--outputDirectory', optimizer_dir, - '--numConcurrentAlgoExecs', config.get('SMAC', 'numConcurrentAlgoExecs'), + '--numConcurrentAlgoExecs', config.get('SMAC', 'num_concurrent_algo_execs'), # '--runGroupName', config.get('SMAC', 'runGroupName'), - '--maxIncumbentRuns', config.get('SMAC', 'maxIncumbentRuns'), + '--maxIncumbentRuns', config.get('SMAC', 'max_incumbent_runs'), '--retryTargetAlgorithmRunCount', - config.get('SMAC', 'retryTargetAlgorithmRunCount'), + config.get('SMAC', 'retry_target_algorithm_run_count'), '--save-runs-every-iteration true', '--intensification-percentage', config.get('SMAC', 'intensification_percentage'), @@ -112,8 +113,8 @@ def build_smac_call(config, options, optimizer_dir): if config.getboolean('SMAC', 'deterministic'): call = " ".join([call, '--deterministic true']) - if config.getboolean('SMAC', 'adaptiveCapping') and \ - config.get('SMAC', 'runObj') == "RUNTIME": + if config.getboolean('SMAC', 'adaptive_capping') and \ + config.get('SMAC', 'run_obj') == "RUNTIME": call = " ".join([call, '--adaptiveCapping true']) if config.getboolean('SMAC', 'rf_full_tree_bootstrap'): @@ -142,7 +143,7 @@ def restore(config, optimizer_dir, **kwargs): # Run SMAC in a manner that it restores the files but then exits fh = open(optimizer_dir + "smac_restart.out", "w") smac_cmd = re.sub('python ' + os.path.dirname(os.path.realpath(__file__)) + - "/" + config.get('SMAC', 'algoExec'), 'pwd', + "/" + config.get('SMAC', 'algo_exec'), 'pwd', kwargs['cmd']) smac_cmd = re.sub('--outputDirectory ' + optimizer_dir, '--outputDirectory ' + optimizer_dir + "restart_rungroups", smac_cmd) @@ -224,12 +225,12 @@ def main(config, options, experiment_dir, **kwargs): # Copy the smac search space and create the instance information fh = open(os.path.join(optimizer_dir, 'train.txt'), "w") - for i in range(config.getint('HPOLIB', 'numberCV')): + for i in range(config.getint('HPOLIB', 'number_cv_folds')): fh.write(str(i) + "\n") fh.close() fh = open(os.path.join(optimizer_dir, 'test.txt'), "w") - for i in range(config.getint('HPOLIB', 'numberCV')): + for i in range(config.getint('HPOLIB', 'number_cv_folds')): fh.write(str(i) + "\n") fh.close() diff --git a/optimizers/smac/smac_2_06_01-devDefault.cfg b/optimizers/smac/smac_2_06_01-devDefault.cfg index 8340c87f..c0d70c79 100644 --- a/optimizers/smac/smac_2_06_01-devDefault.cfg +++ b/optimizers/smac/smac_2_06_01-devDefault.cfg @@ -3,32 +3,33 @@ handles_cv=1 [SMAC] -# cutoffTime = runsolver_time_limit + 100 sec +# cutoff_time = runsolver_time_limit + 100 sec # Set otherwise -# algoExec = %(run_instance)s -# numConcurrentAlgoExecs = %(numberOfConcurrentJobs)s +# algo_exec = %(run_instance)s +# num_concurrent_algo_execs = %(number_of_concurrent_jobs)s # experimentDir = %(exp_directory)s/ # outputDirectory = %(exp_directory)s/smac/output/ -# totalNumRunsLimit = %(numberOfJobs)s * %(cv)s +# total_num_runs_limit = %(number_of_jobs)s * %(cv)s -numRun = 0 -intraInstanceObj = MEAN -runObj = QUALITY -instanceFile = train.txt -testInstanceFile = test.txt +num_run = 0 +intra_instance_obj = MEAN +run_obj = QUALITY +#instance_file = train.txt +#test_instance_file = test.txt #execDir = ./ p = smac_2_06_01-dev/params.pcs rf_full_tree_bootstrap = False rf_split_min = 10 -adaptiveCapping = false -maxIncumbentRuns = 2000 -numIterations = 2147483647 -runtimeLimit = 2147483647 +adaptive_capping = false +max_incumbent_runs = 2000 +num_iterations = 2147483647 +# No one actually cares about it... +runtime_limit = 2147483647 deterministic = True -retryTargetAlgorithmRunCount = 0 +retry_target_algorithm_run_count = 0 intensification_percentage = 0 validation = false diff --git a/optimizers/smac/smac_2_06_01-dev_parser.py b/optimizers/smac/smac_2_06_01-dev_parser.py index 620c7d53..5f9e9a49 100644 --- a/optimizers/smac/smac_2_06_01-dev_parser.py +++ b/optimizers/smac/smac_2_06_01-dev_parser.py @@ -26,19 +26,19 @@ def manipulate_config(config): - if not config.has_option('SMAC', 'cutoffTime'): - config.set('SMAC', 'cutoffTime', + if not config.has_option('SMAC', 'cutoff_time'): + config.set('SMAC', 'cutoff_time', str(config.getint('HPOLIB', 'runsolver_time_limit') + 100)) - if not config.has_option('SMAC', 'algoExec'): - config.set('SMAC', 'algoExec', + if not config.has_option('SMAC', 'algo_exec'): + config.set('SMAC', 'algo_exec', config.get('HPOLIB', 'run_instance')) - if not config.has_option('SMAC', 'totalNumRunsLimit'): - config.set('SMAC', 'totalNumRunsLimit', - str(config.getint('HPOLIB', 'numberOfJobs') * - config.getint('HPOLIB', 'numberCV'))) - if not config.has_option('SMAC', 'numConcurrentAlgoExecs'): - config.set('SMAC', 'numConcurrentAlgoExecs', - config.get('HPOLIB', 'numberOfConcurrentJobs')) + if not config.has_option('SMAC', 'total_num_runs_limit'): + config.set('SMAC', 'total_num_runs_limit', + str(config.getint('HPOLIB', 'number_of_jobs') * + config.getint('HPOLIB', 'number_cv_folds'))) + if not config.has_option('SMAC', 'num_concurrent_algo_execs'): + config.set('SMAC', 'num_concurrent_algo_execs', + config.get('HPOLIB', 'number_of_concurrent_jobs')) path_to_optimizer = config.get('SMAC', 'path_to_optimizer') if not os.path.isabs(path_to_optimizer): diff --git a/optimizers/spearmint/spearmint_april2013_mod.py b/optimizers/spearmint/spearmint_april2013_mod.py index ae08af11..cc52564d 100644 --- a/optimizers/spearmint/spearmint_april2013_mod.py +++ b/optimizers/spearmint/spearmint_april2013_mod.py @@ -39,11 +39,6 @@ def check_dependencies(): - try: - import networkx - except ImportError: - raise ImportError("Networkx cannot be imported. Are you sure it's " - "installed?") try: import google.protobuf except ImportError: @@ -66,7 +61,7 @@ def build_spearmint_call(config, options, optimizer_dir): call = 'python ' + os.path.join(config.get('SPEARMINT', 'path_to_optimizer'), 'spearmint_sync.py') call = ' '.join([call, optimizer_dir, '--config', config.get('SPEARMINT', 'config'), - '--max-concurrent', config.get('HPOLIB', 'numberOfConcurrentJobs'), + '--max-concurrent', config.get('HPOLIB', 'number_of_concurrent_jobs'), '--max-finished-jobs', config.get('SPEARMINT', 'max_finished_jobs'), '--polling-time', config.get('SPEARMINT', 'spearmint_polling_time'), '--grid-size', config.get('SPEARMINT', 'grid_size'), @@ -99,7 +94,7 @@ def restore(config, optimizer_dir, **kwargs): exp_grid = cPickle.load(fh) fh.close() complete_runs = np.sum(exp_grid['status'] == 3) - restored_runs = complete_runs * config.getint('HPOLIB', 'numberCV') + restored_runs = complete_runs * config.getint('HPOLIB', 'number_cv_folds') try: os.remove(os.path.join(optimizer_dir, "expt-grid.pkl.lock")) except OSError: diff --git a/optimizers/spearmint/spearmint_april2013_modDefault.cfg b/optimizers/spearmint/spearmint_april2013_modDefault.cfg index 15599fe5..eed72c5a 100644 --- a/optimizers/spearmint/spearmint_april2013_modDefault.cfg +++ b/optimizers/spearmint/spearmint_april2013_modDefault.cfg @@ -18,7 +18,7 @@ wrapper = 0 spearmint_polling_time = 3.0 #General Parameters max_concurrent = 1 -#max_finished_jobs = %(numberOfJobs)s +#max_finished_jobs = %(number_of_jobs)s # either relative to __file__ or absolute path_to_optimizer = ./spearmint_april2013_mod_src diff --git a/optimizers/spearmint/spearmint_april2013_mod_parser.py b/optimizers/spearmint/spearmint_april2013_mod_parser.py index b36a004f..85df4d02 100644 --- a/optimizers/spearmint/spearmint_april2013_mod_parser.py +++ b/optimizers/spearmint/spearmint_april2013_mod_parser.py @@ -33,7 +33,7 @@ def manipulate_config(config): # GENERAL if not config.has_option('SPEARMINT', 'max_finished_jobs'): config.set('SPEARMINT', 'max_finished_jobs', - config.get('HPOLIB', 'numberOfJobs')) + config.get('HPOLIB', 'number_of_jobs')) path_to_optimizer = config.get('SPEARMINT', 'path_to_optimizer') if not os.path.isabs(path_to_optimizer): diff --git a/optimizers/tpe/hyperopt_august2013_mod.py b/optimizers/tpe/hyperopt_august2013_mod.py index 8e162203..7f71102e 100644 --- a/optimizers/tpe/hyperopt_august2013_mod.py +++ b/optimizers/tpe/hyperopt_august2013_mod.py @@ -64,7 +64,7 @@ def build_tpe_call(config, options, optimizer_dir): call = "python " + os.path.dirname(os.path.realpath(__file__)) + \ "/tpecall.py" call = ' '.join([call, '-p', config.get('TPE', 'space'), - "-m", config.get('TPE', 'numberEvals'), + "-m", config.get('TPE', 'number_evals'), "-s", str(options.seed), "--cwd", optimizer_dir]) if options.restore: @@ -94,7 +94,7 @@ def restore(config, optimizer_dir, **kwargs): # Assumes that all not valid states states are marked crashed if trial['state'] == 2: complete_runs += 1 - restored_runs = complete_runs * config.getint('HPOLIB', 'numberCV') + restored_runs = complete_runs * config.getint('HPOLIB', 'number_cv_folds') return restored_runs diff --git a/optimizers/tpe/hyperopt_august2013_modDefault.cfg b/optimizers/tpe/hyperopt_august2013_modDefault.cfg index df580f1b..09cde416 100644 --- a/optimizers/tpe/hyperopt_august2013_modDefault.cfg +++ b/optimizers/tpe/hyperopt_august2013_modDefault.cfg @@ -1,7 +1,7 @@ [TPE] space = space.py #exp_dir = %(exp_directory)s/tpe/ -#numberEvals = %(numberOfJobs)s +#number_evals = %(number_of_jobs)s # either relative to __file__ or absolute path_to_optimizer = ./hyperopt_august2013_mod_src \ No newline at end of file diff --git a/optimizers/tpe/hyperopt_august2013_mod_parser.py b/optimizers/tpe/hyperopt_august2013_mod_parser.py index 070257f9..418c40b7 100644 --- a/optimizers/tpe/hyperopt_august2013_mod_parser.py +++ b/optimizers/tpe/hyperopt_august2013_mod_parser.py @@ -33,8 +33,8 @@ def manipulate_config(config): if not config.has_option('TPE', 'space'): raise Exception("TPE:space not specified in .cfg") - if not config.has_option('TPE', 'numberEvals'): - config.set('TPE', 'numberEvals', config.get('HPOLIB', 'numberOfJobs')) + if not config.has_option('TPE', 'number_evals'): + config.set('TPE', 'number_evals', config.get('HPOLIB', 'number_of_jobs')) path_to_optimizer = config.get('TPE', 'path_to_optimizer') if not os.path.isabs(path_to_optimizer): diff --git a/optimizers/tpe/random_hyperopt_august2013_mod.py b/optimizers/tpe/random_hyperopt_august2013_mod.py index 6559763c..b11907f7 100644 --- a/optimizers/tpe/random_hyperopt_august2013_mod.py +++ b/optimizers/tpe/random_hyperopt_august2013_mod.py @@ -68,7 +68,7 @@ def build_random_call(config, options, optimizer_dir): call = "python " + os.path.dirname(os.path.realpath(__file__)) + \ "/tpecall.py" call = ' '.join([call, '-p', config.get('TPE', 'space'), - "-m", config.get('TPE', 'numberEvals'), + "-m", config.get('TPE', 'number_evals'), "-s", str(options.seed), "--cwd", optimizer_dir, "--random"]) if options.restore: @@ -93,7 +93,7 @@ def restore(config, optimizer_dir, **kwargs): # Assumes that all states no valid state is marked crashed if trial['state'] == 2: complete_runs += 1 - restored_runs = complete_runs * config.getint('HPOLIB', 'numberCV') + restored_runs = complete_runs * config.getint('HPOLIB', 'number_cv_folds') return restored_runs diff --git a/tests/unittests/branin_test/config.cfg b/tests/unittests/branin_test/config.cfg index 6c6aef57..13017468 100644 --- a/tests/unittests/branin_test/config.cfg +++ b/tests/unittests/branin_test/config.cfg @@ -3,6 +3,6 @@ p = smac/params.txt [DEFAULT] function = ./branin.py -numberOfJobs = 2 +number_of_jobs = 2 result_on_terminate = 1000 -numberCV = 5 +number_cv_folds = 5 diff --git a/tests/unittests/dummy_config.cfg b/tests/unittests/dummy_config.cfg index 90bd6994..196ef165 100644 --- a/tests/unittests/dummy_config.cfg +++ b/tests/unittests/dummy_config.cfg @@ -1,5 +1,5 @@ [HPOLIB] -numberOfJobs = 1 +number_of_jobs = 1 result_on_terminate = 1 function = 1 From 54030da9952288dcc28e8b77d8ebe666ee6a3e20 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Tue, 11 Mar 2014 13:55:43 +0100 Subject: [PATCH 102/139] FIX: issue #34 --- HPOlib/config_parser/generalDefault.cfg | 8 ++++---- HPOlib/runsolver_wrapper.py | 16 ++++++++++------ HPOlib/wrapping.py | 6 +++++- optimizers/smac/smac_2_06_01-dev_parser.py | 7 ++++++- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/HPOlib/config_parser/generalDefault.cfg b/HPOlib/config_parser/generalDefault.cfg index 173a8670..0e273c49 100644 --- a/HPOlib/config_parser/generalDefault.cfg +++ b/HPOlib/config_parser/generalDefault.cfg @@ -11,10 +11,10 @@ number_of_concurrent_jobs = 1 #The mem-limit must be expressed in mega-bytes #The cpu-limit must be expressed in seconds (CPU time) #The time-limit must be expressed in seconds (wall clock time) -runsolver_time_limit = 3600 -total_time_limit = 3600 -memory_limit = 2000 -cpu_limit = 14400 +runsolver_time_limit = +total_time_limit = +memory_limit = +cpu_limit = # E.g. Information for Theano and cuda leading_runsolver_info = diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 16ed5e92..7f4acaad 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -212,9 +212,6 @@ def get_function_filename(cfg): def make_command(cfg, fold, param_string, run_instance_output): - time_limit = cfg.getint('HPOLIB', 'runsolver_time_limit') - memory_limit = cfg.getint('HPOLIB', 'memory_limit') - cpu_limit = cfg.getint('HPOLIB', 'cpu_limit') fn_filename = get_function_filename(cfg) python_cmd = cfg.get("HPOLIB", "leading_algo_info") + " python " + fn_filename python_cmd += " --fold %d --folds %d --params %s" % (fold, cfg.getint( @@ -224,9 +221,16 @@ def make_command(cfg, fold, param_string, run_instance_output): # is flushed to the output directory delay = 0 cmd = cfg.get("HPOLIB", "leading_runsolver_info") - cmd += " runsolver -o %s --timestamp --use-pty -W %d -C %d -M %d -d %d %s" \ - % (run_instance_output, time_limit, cpu_limit, memory_limit, delay, - python_cmd) + cmd += " runsolver -o %s --timestamp --use-pty" % run_instance_output + if cfg.get('HPOLIB', 'runsolver_time_limit'): + cmd += " -W %d" % cfg.getint('HPOLIB', 'runsolver_time_limit') + if cfg.get('HPOLIB', 'cpu_limit'): + cmd += " -C %d" % cfg.getint('HPOLIB', 'cpu_limit') + if cfg.get('HPOLIB', 'memory_limit'): + cmd += " -M %d" % cfg.getint('HPOLIB', 'memory_limit') + if delay is not None: + cmd += " -d %d" % int(delay) + cmd += " " + python_cmd return cmd diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index f5d6b6ac..851d2195 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -272,8 +272,12 @@ def main(): logger.info("-----------------------RUNNING----------------------------------") # http://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python # How often is the experiment pickle supposed to be opened? + if config.get("HPOLIB", "total_time_limit"): + optimizer_end_time = time.time() + config.getint("HPOLIB", "total_time_limit") + else: + optimizer_end_time = sys.float_info.max + last_output = time.time() - optimizer_end_time = time.time() + config.getint("HPOLIB", "total_time_limit") printed_start_configuration = list() printed_end_configuration = list() current_best = -1 diff --git a/optimizers/smac/smac_2_06_01-dev_parser.py b/optimizers/smac/smac_2_06_01-dev_parser.py index 5f9e9a49..b8e5a5c3 100644 --- a/optimizers/smac/smac_2_06_01-dev_parser.py +++ b/optimizers/smac/smac_2_06_01-dev_parser.py @@ -27,8 +27,13 @@ def manipulate_config(config): if not config.has_option('SMAC', 'cutoff_time'): - config.set('SMAC', 'cutoff_time', + print config.get('HPOLIB', 'runsolver_time_limit') + if config.get('HPOLIB', 'runsolver_time_limit'): + config.set('SMAC', 'cutoff_time', str(config.getint('HPOLIB', 'runsolver_time_limit') + 100)) + else: + # SMACs maxint + config.set('SMAC', 'cutoff_time', "2147483647") if not config.has_option('SMAC', 'algo_exec'): config.set('SMAC', 'algo_exec', config.get('HPOLIB', 'run_instance')) From e9064099830ca5e842a94cd261cee6560e3a7f60 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 12 Mar 2014 16:23:18 +0100 Subject: [PATCH 103/139] REFACTOR: issue #27 and clean up function call For some reason I don't remember every target algorithm call started with a call to python which obviously makes no sense if the target algorithm is not written in python. I removed this and all code assuming that the target algorithm is in python and in return added two callbacks, one before the optimizer is called and one afterwards. These callbacks can now be used to check if the function is there, if all dependencies are istalled or delete files after the optimization ... Right now there are no benchmarks which can make use of this, so the feature is not tested at all. --- HPOlib/check_before_start.py | 24 +------------------- HPOlib/config_parser/generalDefault.cfg | 4 +++- HPOlib/runsolver_wrapper.py | 29 ++++++------------------- HPOlib/wrapping.py | 21 +++++++++++++++--- benchmarks/branin/config.cfg | 2 +- benchmarks/har6/config.cfg | 2 +- 6 files changed, 31 insertions(+), 51 deletions(-) diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index 980280a8..a90ed8ad 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -81,21 +81,6 @@ def _check_config(experiment_dir): raise Exception("There is no config.cfg in %s" % experiment_dir) -def _check_function(experiment_dir): - # Check whether path function exists - config_file = os.path.join(experiment_dir, "config.cfg") - config = get_configuration(experiment_dir, None, None) - - path = config.get("HPOLIB", "function") - - if not os.path.isabs(path): - path = os.path.join(experiment_dir, path) - - if not os.path.exists(path): - raise ValueError("Function: %s does not exist" % path) - return - - def check_optimizer(optimizer): path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "optimizers", optimizer) if os.path.isdir(path): @@ -149,11 +134,4 @@ def check_first(experiment_dir): logger.info("..passed") logger.info("Check python_modules..") _check_modules() - logger.info("..passed") - - -def check_second(experiment_dir): - """ Do remaining tests """ - logger.info("Check algorithm..",) - _check_function(experiment_dir) - logger.info("..passed") + logger.info("..passed") \ No newline at end of file diff --git a/HPOlib/config_parser/generalDefault.cfg b/HPOlib/config_parser/generalDefault.cfg index 0e273c49..86a48194 100644 --- a/HPOlib/config_parser/generalDefault.cfg +++ b/HPOlib/config_parser/generalDefault.cfg @@ -5,9 +5,11 @@ use_own_time_measurement = True number_cv_folds = 1 max_crash_per_cv = 3 remove_target_algorithm_output = True - number_of_concurrent_jobs = 1 +function_setup = +function_teardown = + #The mem-limit must be expressed in mega-bytes #The cpu-limit must be expressed in seconds (CPU time) #The time-limit must be expressed in seconds (wall clock time) diff --git a/HPOlib/runsolver_wrapper.py b/HPOlib/runsolver_wrapper.py index 7f4acaad..169ab626 100644 --- a/HPOlib/runsolver_wrapper.py +++ b/HPOlib/runsolver_wrapper.py @@ -193,27 +193,9 @@ def parse_command_line(): return fold, seed -def get_function_filename(cfg): - exp_dir = os.path.dirname(os.getcwd()) - - # Check whether function path is absolute - if os.path.isabs(cfg.get("HPOLIB", "function")): - fn_path = cfg.get("HPOLIB", "function") - else: - # We hope to find function in the experiment directory - fn_path = os.path.join(exp_dir, cfg.get("HPOLIB", "function")) - - if not os.path.exists(fn_path): - logger.critical("%s is not absolute nor in %s. Function cannot be " - "found", cfg.get("HPOLIB", "function"), exp_dir) - sys.exit(1) - - return fn_path - - def make_command(cfg, fold, param_string, run_instance_output): - fn_filename = get_function_filename(cfg) - python_cmd = cfg.get("HPOLIB", "leading_algo_info") + " python " + fn_filename + fn = cfg.get("HPOLIB", "function") + python_cmd = cfg.get("HPOLIB", "leading_algo_info") + " " + fn python_cmd += " --fold %d --folds %d --params %s" % (fold, cfg.getint( "HPOLIB", "number_cv_folds"), param_string) # Do not write the actual task in quotes because runsolver will not work @@ -246,9 +228,12 @@ def parse_output_files(cfg, run_instance_output, runsolver_output_file): runsolver_output_file) result_array, result_string = read_run_instance_output(run_instance_output) if not result_array: - logger.critical("We could not find anything matching our regexp. Last lines of your output:\n%s" + logger.critical("We could not find anything matching our regexp. " + "Setting the target algorithm runtime to the time " + "measured by the runsolver. Last lines of your " + "output:\n%s" % result_string) - instance_wallclock_time = cfg.getfloat("HPOLIB", "runsolver_time_limit") + instance_wallclock_time = measured_wallclock_time result_array = [None]*7 else: instance_wallclock_time = float(result_array[4]) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 851d2195..aacd6a5c 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -191,9 +191,6 @@ def main(): config.set("HPOLIB", "is_not_original_config_file", "True") wrapping_util.save_config_to_file(f, config, write_nones=True) - # _check_function - check_before_start.check_second(experiment_dir) - # initialize/reload pickle file if args.restore: try: @@ -262,6 +259,15 @@ def main(): logger.info(cmd) return 0 else: + # call target_function.setup() + fn_setup = config.get("HPOLIB", "function_setup") + if fn_setup: + try: + output = subprocess.check_output(fn_setup, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + logger.critical(e.output) + sys.exit(1) + logger.info(cmd) output_file = os.path.join(optimizer_dir_in_experiment, optimizer + ".out") fh = open(output_file, "a") @@ -377,6 +383,15 @@ def enqueue_output(out, queue): logger.info("-----------------------END--------------------------------------") fh.close() + # call target_function.teardown() + fn_teardown = config.get("HPOLIB", "function_teardown") + if fn_teardown: + try: + output = subprocess.check_output(fn_teardown, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + logger.critical(e.output) + sys.exit(1) + trials = Experiment.Experiment(optimizer_dir_in_experiment, optimizer) trials.endtime.append(time.time()) #noinspection PyProtectedMember diff --git a/benchmarks/branin/config.cfg b/benchmarks/branin/config.cfg index d7133a8b..20efec78 100644 --- a/benchmarks/branin/config.cfg +++ b/benchmarks/branin/config.cfg @@ -2,6 +2,6 @@ p = smac_2_06_01-dev/params.txt [HPOLIB] -function = branin.py +function = python ../branin.py number_of_jobs = 200 result_on_terminate = 1000 diff --git a/benchmarks/har6/config.cfg b/benchmarks/har6/config.cfg index 3d632034..8ff970e2 100644 --- a/benchmarks/har6/config.cfg +++ b/benchmarks/har6/config.cfg @@ -2,7 +2,7 @@ p = smac_2_06_01-dev/params.txt [HPOLIB] -function = har6.py +function = python ../har6.py number_of_jobs = 200 result_on_terminate = 1 From 2e1a784889beccfd79f7748a4dced2b9cd691fa2 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 13 Mar 2014 13:18:56 +0100 Subject: [PATCH 104/139] ENHANCEMENT: #2 added LaTeX table generation If the user has installed jinja2, he can also use a (so far only static) template. --- HPOlib/Plotting/doAllPlots.py | 29 +++++++++++++++++++++++++++++ HPOlib/Plotting/plot_util.py | 19 +++++++++++++++++++ HPOlib/Plotting/statistics.py | 29 ++++++++--------------------- 3 files changed, 56 insertions(+), 21 deletions(-) diff --git a/HPOlib/Plotting/doAllPlots.py b/HPOlib/Plotting/doAllPlots.py index d48de251..b5a1c84b 100644 --- a/HPOlib/Plotting/doAllPlots.py +++ b/HPOlib/Plotting/doAllPlots.py @@ -117,6 +117,23 @@ def _box_whisker(pkl_list, name_list, save): sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) +def _generate_tex_table(pkl_list, name_list, save): + plotting_dir = os.path.dirname(os.path.realpath(__file__)) + cur_dir = os.getcwd() + # noinspection PyBroadException + try: + os.chdir(plotting_dir) + import generateTexTable + table = generateTexTable.main(pkl_list, name_list) + with open(save, "w") as fh: + fh.write(table) + os.chdir(cur_dir) + sys.stdout.write("passed\n") + except Exception, e: + sys.stderr.write(format_traceback(sys.exc_info())) + sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) + + def _statistics(pkl_list, name_list, save=""): plotting_dir = os.path.dirname(os.path.realpath(__file__)) cur_dir = os.getcwd() @@ -171,6 +188,9 @@ def main(): pkl_list, name_list = plot_util.get_pkl_and_name_list(unknown) + if len(pkl_list) == 0: + raise ValueError("You must at least provide one experiment pickle.") + time_str = int(time.time() % 1000) if not os.path.isdir(save_dir) and save_dir is not "": @@ -210,6 +230,15 @@ def main(): sys.stdout.write("statistics.py ... %s ..." % tmp_save) _statistics(pkl_list=pkl_list, name_list=name_list, save=tmp_save) + # LaTeX table + if save_dir is not "": + tmp_save = os.path.join(save_dir, "table_%s.tex" % time_str) + else: + tmp_save = save_dir + sys.stdout.write("generateTexTable.py ... %s ..." % tmp_save) + _generate_tex_table(pkl_list, name_list, tmp_save) + + # We can always plot this # OptimizerOverhead if save_dir is not "": diff --git a/HPOlib/Plotting/plot_util.py b/HPOlib/Plotting/plot_util.py index 97514bd2..47ce9e99 100644 --- a/HPOlib/Plotting/plot_util.py +++ b/HPOlib/Plotting/plot_util.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import cPickle import os import numpy as np import sys @@ -24,6 +25,24 @@ __contact__ = "automl.org" +def read_pickles(name_list, pkl_list, cut=sys.maxint): + best_dict = dict() + idx_dict = dict() + keys = list() + for i in range(len(name_list)): + keys.append(name_list[i][0]) + best_dict[name_list[i][0]] = list() + idx_dict[name_list[i][0]] = list() + for pkl in pkl_list[i]: + fh = open(pkl) + trial = cPickle.load(fh) + fh.close() + best, idx = get_best_value_and_index(trial, cut) + best_dict[name_list[i][0]].append(best) + idx_dict[name_list[i][0]].append(idx) + return best_dict, idx_dict, keys + + def get_pkl_and_name_list(argument_list): name_list = list() pkl_list = list() diff --git a/HPOlib/Plotting/statistics.py b/HPOlib/Plotting/statistics.py index 09004391..96bb3fbb 100644 --- a/HPOlib/Plotting/statistics.py +++ b/HPOlib/Plotting/statistics.py @@ -115,27 +115,13 @@ def _mann_whitney_u(x, y=None): def main(pkl_list, name_list): - - best_dict = dict() - idx_dict = dict() - keys = list() - - for i in range(len(name_list)): - keys.append(name_list[i][0]) - best_dict[name_list[i][0]] = list() - idx_dict[name_list[i][0]] = list() - for pkl in pkl_list[i]: - fh = open(pkl) - trial = cPickle.load(fh) - fh.close() - best, idx = plot_util.get_best_value_and_index(trial, args.cut) - best_dict[name_list[i][0]].append(best) - idx_dict[name_list[i][0]].append(idx) + best_dict, idx_dict, keys = plot_util.read_pickles(name_list, pkl_list) for k in keys: sys.stdout.write("%10s: %s experiment(s)\n" % (k, len(best_dict[k]))) sys.stdout.write("Unpaired t-tests-----------------------------------------------------\n") + # TODO: replace by itertools for idx, k in enumerate(keys): if len(keys) > 1: for jdx, j in enumerate(keys[idx+1:]): @@ -143,26 +129,27 @@ def main(pkl_list, name_list): rounded_t_true, rounded_p_true = stats.ttest_ind(numpy.round(best_dict[k], 3), numpy.round(best_dict[j], 3)) + sys.stdout.write("%10s vs %10s\n" % (k, j)) sys.stdout.write("Standard independent 2 sample test, equal population variance\n") - sys.stdout.write("%10s vs %10s" % (k, j)) - sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%) \n" % + sys.stdout.write(" "*24 + " T: %10.5e, p-value: %10.5e (%5.3f%%) \n" % (t_true, p_true, p_true*100)) sys.stdout.write("Rounded: ") - sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" % + sys.stdout.write(" T: %10.5e, p-value: %10.5e (%5.3f%%)\n" % (rounded_t_true, rounded_p_true, rounded_p_true*100)) if StrictVersion(scipy.__version__) >= StrictVersion('0.11.0'): - print scipy.__version__ >= '0.11.0' + # print scipy.__version__ >= '0.11.0' t_false, p_false = stats.ttest_ind(best_dict[k], best_dict[j], equal_var=False) rounded_t_false, rounded_p_false = stats.ttest_ind(numpy.round(best_dict[k], 3), numpy.round(best_dict[j], 3), equal_var=False) sys.stdout.write("Welch's t-test, no equal population variance\n") - sys.stdout.write("%10s vs %10s" % (k, j)) + sys.stdout.write(" "*24) sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" % (t_false, p_false, p_false*100)) sys.stdout.write("Rounded: ") sys.stdout.write(": T: %10.5e, p-value: %10.5e (%5.3f%%)\n" % (rounded_t_false, rounded_p_false, rounded_p_false*100)) + sys.stdout.write("\n") sys.stdout.write("Best Value-----------------------------------------------------------\n") for k in keys: From c5dd2377ad1f5f8255ef86801de05135f2d5f143 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 13 Mar 2014 13:20:55 +0100 Subject: [PATCH 105/139] FIX #2 forgot the actual file --- HPOlib/Plotting/generateTexTable.py | 139 ++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 HPOlib/Plotting/generateTexTable.py diff --git a/HPOlib/Plotting/generateTexTable.py b/HPOlib/Plotting/generateTexTable.py new file mode 100644 index 00000000..47e27ff7 --- /dev/null +++ b/HPOlib/Plotting/generateTexTable.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python + +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from argparse import ArgumentParser +from collections import OrderedDict +from StringIO import StringIO +import numpy as np +import sys + +try: + import jinja2 + from jinja2 import Template +except ImportError: + jinja2 = "" + +from HPOlib.Plotting import plot_util + +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" + + +template_string = \ +"""\\documentclass{article} % For LaTeX2 + +\\usepackage{multirow} % import command \multicolmun +\\usepackage{tabularx} % Convenient table formatting +\\usepackage{booktabs} % provides \toprule, \midrule and \bottomrule + +\\begin{document} + +\\begin{table}[t] +\\begin{tabularx}{\\textwidth}{lr|Xr|Xr|Xr} +\\toprule +\multicolumn{2}{l}{} +{%- for name in result_values -%} +&\multicolumn{2}{c}{\\bf {{name}}} +{%- endfor -%} +\\\\ +\\multicolumn{1}{l}{\\bf Experiment} &\multicolumn{1}{r}{\\#evals} +{%- for name in result_values -%} +&\\multicolumn{1}{l}{Valid.\\ loss} &\\multicolumn{1}{r}{Best loss} +{%- endfor -%} +\\\\ +\\toprule +{{ experiment }} & {{ evals }} +{%- for name in result_values -%} +{%- set results = result_values[name] -%} +{{ ' & ' }}{{ results['mean'] }}$\\pm${{ results['std'] }} & {{ results['min'] }} +{%- endfor %} \\\\ +\\bottomrule +\\end{tabularx} +\\end{table} +\\end{document} +""" + +def main(pkl_list, name_list, cut=sys.maxint): + tex = StringIO() + result_values = OrderedDict([(name[0], dict()) for name in name_list]) + + best_dict, idx_dict, keys = plot_util.read_pickles(name_list, pkl_list, cut) + + for name in result_values: + values = result_values[name] + + values["mean"] = np.mean(best_dict[name]) + values["mean_bold"] = False + values["mean_italic"] = False + + values["std"] = np.std(best_dict[name]) + values["std_bold"] = False + values["std_italic"] = False + + values["min"] = np.min(best_dict[name]) + values["min_bold"] = False + values["min_italic"] = False + + values["max"] = np.min(best_dict[name]) + values["max_bold"] = False + values["max_italic"] = False + + if jinja2: + template = Template(template_string) + tex.write(template.render(result_values=result_values, + experiment="Name", evals="\\#evals")) + else: + tex.write("Name & #evals") + for name in result_values: + values = result_values[name] + tex.write(" & ") + tex.write(values["mean"]) + tex.write("$\\pm$") + tex.write(values["std"]) + tex.write(" & ") + tex.write(values["min"]) + tex.write("\\\\") + + tex.seek(0) + table = tex.getvalue() + print table + return table + + +if __name__ == "__main__": + prog = "python statistics.py WhatIsThis WhatIsThis [WhatIsThis ]" + description = "Generate a LaTeX results table." + + parser = ArgumentParser(description=description, prog=prog) + + parser.add_argument("-c", "--c", dest="cut", default=sys.maxint, + type=int, help="Only consider that many evaluations") + args, unknown = parser.parse_known_args() + # TODO-list: + # 1. Add statistical relevance + # 2. Add parameters to control whether a value should be printed bold + # face, underlined or whatever + # 3. Add a custom rounding + # 4. Determine the experiment name and number of evaluations + + pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) + main(pkl_list_main, name_list_main, args.cut) + + From 7c98159c113569283417ca1da2130f2a469f778b Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Thu, 13 Mar 2014 15:20:52 +0100 Subject: [PATCH 106/139] MINOR CHANGE: Use unused imports (random/tpe) check_dependencies prints in logger.debug the versions of the required python modules --- optimizers/tpe/hyperopt_august2013_mod.py | 6 ++++++ optimizers/tpe/random_hyperopt_august2013_mod.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/optimizers/tpe/hyperopt_august2013_mod.py b/optimizers/tpe/hyperopt_august2013_mod.py index 7f71102e..dd05d3e2 100644 --- a/optimizers/tpe/hyperopt_august2013_mod.py +++ b/optimizers/tpe/hyperopt_august2013_mod.py @@ -30,30 +30,36 @@ __contact__ = "automl.org" +# noinspection PyUnresolvedReferences def check_dependencies(): try: import nose + logger.debug("\tNose: %d\n" % str(nose.__version__)) except ImportError: raise ImportError("Nose cannot be imported. Are you sure it's " "installed?") try: import networkx + logger.debug("\tnetworkx: %d\n" % str(networkx.__version__)) except ImportError: raise ImportError("Networkx cannot be imported. Are you sure it's " "installed?") try: import pymongo + logger.debug("\tpymongo: %d\n" % str(pymongo.__version__)) from bson.objectid import ObjectId except ImportError: raise ImportError("Pymongo cannot be imported. Are you sure it's" " installed?") try: import numpy + logger.debug("\tnumpy: %d" % str(numpy.__version__)) except ImportError: raise ImportError("Numpy cannot be imported. Are you sure that it's" " installed?") try: import scipy + logger.debug("\tscipy: %d" % str(scipy.__version__)) except ImportError: raise ImportError("Scipy cannot be imported. Are you sure that it's" " installed?") diff --git a/optimizers/tpe/random_hyperopt_august2013_mod.py b/optimizers/tpe/random_hyperopt_august2013_mod.py index b11907f7..9a485ea4 100644 --- a/optimizers/tpe/random_hyperopt_august2013_mod.py +++ b/optimizers/tpe/random_hyperopt_august2013_mod.py @@ -38,27 +38,32 @@ def check_dependencies(): try: import nose + logger.debug("\tNose: %d\n" % str(nose.__version__)) except ImportError: raise ImportError("Nose cannot be imported. Are you sure it's " "installed?") try: import networkx + logger.debug("\tnetworkx: %d\n" % str(networkx.__version__)) except ImportError: raise ImportError("Networkx cannot be imported. Are you sure it's " "installed?") try: import pymongo + logger.debug("\tpymongo: %d\n" % str(pymongo.__version__)) from bson.objectid import ObjectId except ImportError: raise ImportError("Pymongo cannot be imported. Are you sure it's" " installed?") try: import numpy + logger.debug("\tnumpy: %d" % str(numpy.__version__)) except ImportError: raise ImportError("Numpy cannot be imported. Are you sure that it's" " installed?") try: import scipy + logger.debug("\tscipy: %d" % str(scipy.__version__)) except ImportError: raise ImportError("Scipy cannot be imported. Are you sure that it's" " installed?") From 7837b7cd7f210c937f065df35cbc7d7f63c808a9 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Thu, 13 Mar 2014 15:30:39 +0100 Subject: [PATCH 107/139] MINOR CHANGE: Remove unused imports --- HPOlib/check_before_start.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index a90ed8ad..113c3f1e 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -23,20 +23,15 @@ import subprocess import sys -from HPOlib.wrapping_util import get_configuration - - logger = logging.getLogger("HPOlib.check_before_start") -"""This script checks whether all dependencies are installed""" - - def _check_runsolver(): # check whether runsolver is in path process = subprocess.Popen("which runsolver", stdout=subprocess.PIPE, - stderr=subprocess.PIPE, shell=True, executable="/bin/bash") - stdoutdata, stderrdata = process.communicate() + stderr=subprocess.PIPE, shell=True, + executable="/bin/bash") + stdoutdata, _stderrdata = process.communicate() if stdoutdata is not None and "runsolver" in stdoutdata: pass @@ -46,8 +41,9 @@ def _check_runsolver(): "Your $PATH is: " + os.environ['PATH']) -# noinspection PyUnresolvedReferences def _check_modules(): + """Checks whether all dependencies are installed""" + try: import numpy if numpy.__version__ < "1.6.0": @@ -66,6 +62,7 @@ def _check_modules(): try: import theano + logger.debug("\tTheano: %d" % str(theano.__version__)) except ImportError: logger.warning("Theano not found. You might need this to run some " "more complex benchmarks!") From 6c72a3db38b034a9bb1bd390a2374995b5dd37a8 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Thu, 13 Mar 2014 15:55:06 +0100 Subject: [PATCH 108/139] MINOR CHANGE --- HPOlib/Plotting/statistics.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/HPOlib/Plotting/statistics.py b/HPOlib/Plotting/statistics.py index 96bb3fbb..8ad01563 100644 --- a/HPOlib/Plotting/statistics.py +++ b/HPOlib/Plotting/statistics.py @@ -21,10 +21,8 @@ from argparse import ArgumentParser import cPickle import sys -from distutils.version import StrictVersion import numpy -from numpy.ma import asarray import scipy from scipy import stats @@ -124,7 +122,7 @@ def main(pkl_list, name_list): # TODO: replace by itertools for idx, k in enumerate(keys): if len(keys) > 1: - for jdx, j in enumerate(keys[idx+1:]): + for j in keys[idx+1:]: t_true, p_true = stats.ttest_ind(best_dict[k], best_dict[j]) rounded_t_true, rounded_p_true = stats.ttest_ind(numpy.round(best_dict[k], 3), numpy.round(best_dict[j], 3)) @@ -136,7 +134,7 @@ def main(pkl_list, name_list): sys.stdout.write("Rounded: ") sys.stdout.write(" T: %10.5e, p-value: %10.5e (%5.3f%%)\n" % (rounded_t_true, rounded_p_true, rounded_p_true*100)) - if StrictVersion(scipy.__version__) >= StrictVersion('0.11.0'): + if tuple(map(int, (scipy.__version__.split(".")))) >= (0, 11, 0): # print scipy.__version__ >= '0.11.0' t_false, p_false = stats.ttest_ind(best_dict[k], best_dict[j], equal_var=False) rounded_t_false, rounded_p_false = stats.ttest_ind(numpy.round(best_dict[k], 3), @@ -153,21 +151,15 @@ def main(pkl_list, name_list): sys.stdout.write("Best Value-----------------------------------------------------------\n") for k in keys: - std_best = numpy.std(best_dict[k]) - min_best = numpy.min(best_dict[k]) - max_best = numpy.max(best_dict[k]) - mean_best = numpy.mean(best_dict[k]) sys.stdout.write("%10s: %10.5f (min: %10.5f, max: %10.5f, std: %5.3f)\n" % - (k, float(mean_best), float(min_best), max_best, float(std_best))) + (k, float(numpy.mean(best_dict[k])), float(numpy.min(best_dict[k])), + numpy.max(best_dict[k]), float(numpy.std(best_dict[k])))) sys.stdout.write("Needed Trials--------------------------------------------------------\n") for k in keys: - std_idx = numpy.std(idx_dict[k]) - mean_idx = numpy.mean(idx_dict[k]) - min_idx = numpy.min(idx_dict[k]) - max_idx = numpy.max(idx_dict[k]) sys.stdout.write("%10s: %10.5f (min: %10.5f, max: %10.5f, std: %5.3f)\n" % - (k, float(mean_idx), float(min_idx), max_idx, float(std_idx))) + (k, float(numpy.mean(idx_dict[k])), float(numpy.min(idx_dict[k])), + numpy.max(idx_dict[k]), float(numpy.std(idx_dict[k])))) sys.stdout.write("------------------------------------------------------------------------\n") From 604f3407fa2d91b66997beceb4dc17155b3af139 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Fri, 14 Mar 2014 10:19:28 +0100 Subject: [PATCH 109/139] FIX print formatting --- optimizers/tpe/hyperopt_august2013_mod.py | 10 +++++----- optimizers/tpe/random_hyperopt_august2013_mod.py | 11 ++++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/optimizers/tpe/hyperopt_august2013_mod.py b/optimizers/tpe/hyperopt_august2013_mod.py index dd05d3e2..f55761f4 100644 --- a/optimizers/tpe/hyperopt_august2013_mod.py +++ b/optimizers/tpe/hyperopt_august2013_mod.py @@ -34,32 +34,32 @@ def check_dependencies(): try: import nose - logger.debug("\tNose: %d\n" % str(nose.__version__)) + logger.debug("\tNose: %s\n" % str(nose.__version__)) except ImportError: raise ImportError("Nose cannot be imported. Are you sure it's " "installed?") try: import networkx - logger.debug("\tnetworkx: %d\n" % str(networkx.__version__)) + logger.debug("\tnetworkx: %s\n" % str(networkx.__version__)) except ImportError: raise ImportError("Networkx cannot be imported. Are you sure it's " "installed?") try: import pymongo - logger.debug("\tpymongo: %d\n" % str(pymongo.__version__)) + logger.debug("\tpymongo: %s\n" % str(pymongo.__version__)) from bson.objectid import ObjectId except ImportError: raise ImportError("Pymongo cannot be imported. Are you sure it's" " installed?") try: import numpy - logger.debug("\tnumpy: %d" % str(numpy.__version__)) + logger.debug("\tnumpy: %s" % str(numpy.__version__)) except ImportError: raise ImportError("Numpy cannot be imported. Are you sure that it's" " installed?") try: import scipy - logger.debug("\tscipy: %d" % str(scipy.__version__)) + logger.debug("\tscipy: %s" % str(scipy.__version__)) except ImportError: raise ImportError("Scipy cannot be imported. Are you sure that it's" " installed?") diff --git a/optimizers/tpe/random_hyperopt_august2013_mod.py b/optimizers/tpe/random_hyperopt_august2013_mod.py index 9a485ea4..851e6b09 100644 --- a/optimizers/tpe/random_hyperopt_august2013_mod.py +++ b/optimizers/tpe/random_hyperopt_august2013_mod.py @@ -38,37 +38,38 @@ def check_dependencies(): try: import nose - logger.debug("\tNose: %d\n" % str(nose.__version__)) + logger.debug("\tNose: %s\n" % str(nose.__version__)) except ImportError: raise ImportError("Nose cannot be imported. Are you sure it's " "installed?") try: import networkx - logger.debug("\tnetworkx: %d\n" % str(networkx.__version__)) + logger.debug("\tnetworkx: %s\n" % str(networkx.__version__)) except ImportError: raise ImportError("Networkx cannot be imported. Are you sure it's " "installed?") try: import pymongo - logger.debug("\tpymongo: %d\n" % str(pymongo.__version__)) + logger.debug("\tpymongo: %s\n" % str(pymongo.__version__)) from bson.objectid import ObjectId except ImportError: raise ImportError("Pymongo cannot be imported. Are you sure it's" " installed?") try: import numpy - logger.debug("\tnumpy: %d" % str(numpy.__version__)) + logger.debug("\tnumpy: %s" % str(numpy.__version__)) except ImportError: raise ImportError("Numpy cannot be imported. Are you sure that it's" " installed?") try: import scipy - logger.debug("\tscipy: %d" % str(scipy.__version__)) + logger.debug("\tscipy: %s" % str(scipy.__version__)) except ImportError: raise ImportError("Scipy cannot be imported. Are you sure that it's" " installed?") + def build_random_call(config, options, optimizer_dir): call = "python " + os.path.dirname(os.path.realpath(__file__)) + \ "/tpecall.py" From f5f4d7ba7ddc58c91873afb59aa2803309468841 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Fri, 14 Mar 2014 10:44:04 +0100 Subject: [PATCH 110/139] FIX in check dependencies --- optimizers/tpe/hyperopt_august2013_mod.py | 2 +- optimizers/tpe/random_hyperopt_august2013_mod.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/optimizers/tpe/hyperopt_august2013_mod.py b/optimizers/tpe/hyperopt_august2013_mod.py index f55761f4..bc5ee359 100644 --- a/optimizers/tpe/hyperopt_august2013_mod.py +++ b/optimizers/tpe/hyperopt_august2013_mod.py @@ -46,7 +46,7 @@ def check_dependencies(): "installed?") try: import pymongo - logger.debug("\tpymongo: %s\n" % str(pymongo.__version__)) + logger.debug("\tpymongo: %s\n" % str(pymongo.version)) from bson.objectid import ObjectId except ImportError: raise ImportError("Pymongo cannot be imported. Are you sure it's" diff --git a/optimizers/tpe/random_hyperopt_august2013_mod.py b/optimizers/tpe/random_hyperopt_august2013_mod.py index 851e6b09..1d35ebcf 100644 --- a/optimizers/tpe/random_hyperopt_august2013_mod.py +++ b/optimizers/tpe/random_hyperopt_august2013_mod.py @@ -50,7 +50,7 @@ def check_dependencies(): "installed?") try: import pymongo - logger.debug("\tpymongo: %s\n" % str(pymongo.__version__)) + logger.debug("\tpymongo: %s\n" % str(pymongo.version)) from bson.objectid import ObjectId except ImportError: raise ImportError("Pymongo cannot be imported. Are you sure it's" From fa12057a3b87f8315d69e86e343212205067728f Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Fri, 14 Mar 2014 10:57:18 +0100 Subject: [PATCH 111/139] FIX: Updated tests --- tests/unittests/test_runsolver_wrapper.py | 3 ++- tests/unittests/test_wrapping_util.py | 32 +++++++++++------------ 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/tests/unittests/test_runsolver_wrapper.py b/tests/unittests/test_runsolver_wrapper.py index bd4621f5..198f0fe9 100644 --- a/tests/unittests/test_runsolver_wrapper.py +++ b/tests/unittests/test_runsolver_wrapper.py @@ -89,7 +89,8 @@ def test_read_runsolver_output_warning(self): def test_read_run_instance_output_no_result(self): result_array, result_string = runsolver_wrapper.\ read_run_instance_output("run_instance_no_result.txt") - self.assertTrue(result_string is None) + # We expect some useful output for the user + self.assertFalse(result_string is None) self.assertTrue(result_array is None) def test_read_run_instance_output_result(self): diff --git a/tests/unittests/test_wrapping_util.py b/tests/unittests/test_wrapping_util.py index d4828a3d..61bb28d4 100644 --- a/tests/unittests/test_wrapping_util.py +++ b/tests/unittests/test_wrapping_util.py @@ -28,33 +28,32 @@ import HPOlib.wrapping_util as wrapping_util import HPOlib.config_parser.parse as parse + class WrappingTestUtil(unittest.TestCase): def setUp(self): # Change into the test directory os.chdir(os.path.dirname(os.path.realpath(__file__))) # Make sure there is no config file + # noinspection PyBroadException try: os.remove("./config.cfg") except: pass - def test_save_config_to_file(self): - unknown = ['--HPOLIB:total_time_limit', '50', - '--HPOLIB:numberofjobs', '2'] config = parse.parse_config("dummy_config.cfg", allow_no_value=True) config.set("HPOLIB", "total_time_limit", None) string_stream = StringIO.StringIO() wrapping_util.save_config_to_file(string_stream, config) file_content = string_stream.getvalue() asserted_file_content = "[HPOLIB]\n" \ - "numberofjobs = 1\n" \ - "result_on_terminate = 1\n"\ - "function = 1\n"\ - "total_time_limit = None\n"\ - "[GRIDSEARCH]\n" \ - "params = params.pcs\n" + "number_of_jobs = 1\n" \ + "result_on_terminate = 1\n"\ + "function = 1\n"\ + "total_time_limit = None\n"\ + "[GRIDSEARCH]\n" \ + "params = params.pcs\n" self.assertEqual(asserted_file_content, file_content) string_stream.close() @@ -67,23 +66,22 @@ def test_save_config_to_file_ignore_none(self): write_nones=False) file_content = string_stream.getvalue() asserted_file_content = "[HPOLIB]\n" \ - "numberofjobs = 1\n" \ - "result_on_terminate = 1\n"\ - "function = 1\n"\ - "[GRIDSEARCH]\n" \ - "params = params.pcs\n" + "number_of_jobs = 1\n" \ + "result_on_terminate = 1\n"\ + "function = 1\n"\ + "[GRIDSEARCH]\n" \ + "params = params.pcs\n" self.assertEqual(asserted_file_content, file_content) string_stream.close() def test_use_option_parser_with_config(self): sys.argv = ['wrapping.py', '-s', '1', '-t', 'DBNet', '-o', 'SMAC', - '--HPOLIB:numberofjobs', '2'] + '--HPOLIB:number_of_jobs', '2'] args, unknown = wrapping.use_arg_parser() self.assertEqual(len(unknown), 2) config = ConfigParser.SafeConfigParser(allow_no_value=True) config.read("dummy_config.cfg") config_args = wrapping_util.parse_config_values_from_unknown_arguments( unknown, config) - self.assertEqual(vars(config_args)['HPOLIB:numberofjobs'], - '2') + self.assertEqual(vars(config_args)['HPOLIB:number_of_jobs'], '2') From 5ab2872888e0f3d22584c295e2df3c067de86234 Mon Sep 17 00:00:00 2001 From: Tobias Domhan Date: Tue, 18 Mar 2014 11:34:25 +0100 Subject: [PATCH 112/139] fixed issue with how missing values are filled in plotTraceWithStd_perTime. --- HPOlib/Plotting/plotTraceWithStd_perTime.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HPOlib/Plotting/plotTraceWithStd_perTime.py b/HPOlib/Plotting/plotTraceWithStd_perTime.py index 78f1f95a..b2ee28e2 100644 --- a/HPOlib/Plotting/plotTraceWithStd_perTime.py +++ b/HPOlib/Plotting/plotTraceWithStd_perTime.py @@ -134,10 +134,10 @@ def fill_trajectories(trace_list, times_list): counter = [1 for i in range(number_exp)] finish = False - # We need to insert the max values + # We need to insert the max values in the beginning and the min values in the end for i in range(number_exp): trace_list[i].insert(0, max_value) - trace_list[i].append(0) + trace_list[i].append(np.min(trace_list[i])) times_list[i].insert(0, 0) times_list[i].append(sys.maxint) From b66b5a3334af82936b4e0f2a31177bcf4a3ba3f4 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Wed, 19 Mar 2014 17:00:29 +0100 Subject: [PATCH 113/139] REFACTOR: Adding new colors to plot --- HPOlib/Plotting/plotBranin.py | 8 ++++++-- HPOlib/Plotting/plotTraceWithStd_perTime.py | 9 +++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/HPOlib/Plotting/plotBranin.py b/HPOlib/Plotting/plotBranin.py index cf8e79f9..b5a7fb49 100644 --- a/HPOlib/Plotting/plotBranin.py +++ b/HPOlib/Plotting/plotBranin.py @@ -80,8 +80,12 @@ def plot_contour(trial_list, name_list, save="", title=""): x = np.zeros(len(trial_list[opt]["trials"])) y = np.zeros(len(trial_list[opt]["trials"])) for i in range(len(x)): - x[i] = float(trial_list[opt]["trials"][i]["params"]["-x"].strip("'")) - y[i] = float(trial_list[opt]["trials"][i]["params"]["-y"].strip("'")) + if '-x' in trial_list[opt]["trials"][i]["params"]: + x[i] = float(trial_list[opt]["trials"][i]["params"]["-x"].strip("'")) + y[i] = float(trial_list[opt]["trials"][i]["params"]["-y"].strip("'")) + else: + x[i] = float(trial_list[opt]["trials"][i]["params"]["x"].strip("'")) + y[i] = float(trial_list[opt]["trials"][i]["params"]["y"].strip("'")) matplotlib.pyplot.scatter(x[0:10], y[0:10], marker=m, s=size, facecolors=c, linewidth=0.1) matplotlib.pyplot.scatter(x[10:-10], y[10:-10], marker=m, diff --git a/HPOlib/Plotting/plotTraceWithStd_perTime.py b/HPOlib/Plotting/plotTraceWithStd_perTime.py index b2ee28e2..c91b9266 100644 --- a/HPOlib/Plotting/plotTraceWithStd_perTime.py +++ b/HPOlib/Plotting/plotTraceWithStd_perTime.py @@ -35,8 +35,13 @@ def plot_optimization_trace(trial_list, name_list, times_list, optimum=0, title="", log=True, save="", y_max=0, y_min=0, scale_std=1): - # markers = itertools.cycle(['o', 's', 'x', '^']) - colors = itertools.cycle(['b', 'g', 'r', 'k']) + markers = itertools.cycle(['o', 's', 'x']) + colors = itertools.cycle(['b', 'Navy', 'RoyalBlue', + 'g', 'DarkGreen', 'SpringGreen', + 'r', 'Maroon', 'LightCoral']) + colors_alt = itertools.cycle(['Navy', 'RoyalBlue', + 'DarkGreen', 'SpringGreen', + 'Maroon', 'LightCoral']) linestyles = itertools.cycle(['-']) size = 1 From 21b7e08ab892813c0058da0eab776e572801b9ea Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Thu, 20 Mar 2014 10:12:28 +0100 Subject: [PATCH 114/139] REFACTOR: Replace np.nanmean/scipy.nanmean replace two occurences of nanmean, because nanmean does not work in older numpy/scipy versions. Replace with: np.nansum(arr) / (arr.size - np.count_nonzero(np.isfinite(arr))) --- HPOlib/Experiment.py | 5 ++++- HPOlib/wrapping.py | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/HPOlib/Experiment.py b/HPOlib/Experiment.py index 1dbdc1b4..641ef1c4 100644 --- a/HPOlib/Experiment.py +++ b/HPOlib/Experiment.py @@ -180,7 +180,10 @@ def get_arg_best(self): if np.isfinite(trial['result']): tmp_res = trial['result'] elif np.isfinite(trial['instance_results']).any(): - tmp_res = scipy.nanmean(trial['instance_results']) + tmp_res = np.nansum(trial['instance_results']) / \ + (trial['instance_results'].size - np.count_nonzero(np.isfinite(trial['instance_results']))) + # np.nanmean is not available in older numpy versions + # tmp_res = scipy.nanmean(trial['instance_results']) else: continue if tmp_res < best_value: diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index aacd6a5c..c7869a57 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -366,8 +366,11 @@ def enqueue_output(out, queue): current_best = tmp_current_best res = trials.trials[current_best]["result"] if \ np.isfinite(trials.trials[current_best]["result"]) \ - else np.nanmean(trials.trials[current_best][ - "instance_results"]) + else np.nansum(trials.trials[current_best]["instance_results"]) / \ + (trials.trials[current_best]["instance_results"].size - + np.count_nonzero(np.isfinite(trials.trials[current_best]["instance_results"]))) + #np.nanmean(trials.trials[current_best]["instance_results"]) + # nanmean does not work for all numpy version logger.info("Result %10f, current best %10f", last_result, res) printed_end_configuration.append(i) From feb02e4b7f6eda2000635251104b346bf6dd28bf Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Thu, 20 Mar 2014 10:42:39 +0100 Subject: [PATCH 115/139] FIX: Correct nanmean replace --- HPOlib/Experiment.py | 2 +- HPOlib/wrapping.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/HPOlib/Experiment.py b/HPOlib/Experiment.py index 641ef1c4..f61069b2 100644 --- a/HPOlib/Experiment.py +++ b/HPOlib/Experiment.py @@ -181,7 +181,7 @@ def get_arg_best(self): tmp_res = trial['result'] elif np.isfinite(trial['instance_results']).any(): tmp_res = np.nansum(trial['instance_results']) / \ - (trial['instance_results'].size - np.count_nonzero(np.isfinite(trial['instance_results']))) + (trial['instance_results'].size - np.count_nonzero(~np.isfinite(trial['instance_results']))) # np.nanmean is not available in older numpy versions # tmp_res = scipy.nanmean(trial['instance_results']) else: diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index c7869a57..23eb7518 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -364,11 +364,14 @@ def enqueue_output(out, queue): tmp_current_best = trials.get_arg_best() if tmp_current_best <= i: current_best = tmp_current_best - res = trials.trials[current_best]["result"] if \ - np.isfinite(trials.trials[current_best]["result"]) \ - else np.nansum(trials.trials[current_best]["instance_results"]) / \ - (trials.trials[current_best]["instance_results"].size - - np.count_nonzero(np.isfinite(trials.trials[current_best]["instance_results"]))) + # Calculate current best + # Check if last result is finite, if not calc nanmean over all instances + dct_helper = trials.trials[current_best] + res = dct_helper["result"] if \ + np.isfinite(dct_helper["result"]) \ + else sum([ele for ele in dct_helper["instance_results"] if np.isfinite(ele)]) / \ + (dct_helper["instance_results"].size - + np.count_nonzero(~np.isfinite(dct_helper["instance_results"]))) #np.nanmean(trials.trials[current_best]["instance_results"]) # nanmean does not work for all numpy version logger.info("Result %10f, current best %10f", From d5cfbf910ee6e9aa91c96b8bcaa77b9ad3d4d45b Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Thu, 20 Mar 2014 11:27:12 +0100 Subject: [PATCH 116/139] REFACTOR: Add nan_mean to wrapping_util and add unittest --- HPOlib/Experiment.py | 4 ++-- HPOlib/wrapping.py | 4 +--- HPOlib/wrapping_util.py | 15 +++++++++++++++ tests/unittests/test_wrapping_util.py | 16 +++++++++++++++- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/HPOlib/Experiment.py b/HPOlib/Experiment.py index f61069b2..0c64b64e 100644 --- a/HPOlib/Experiment.py +++ b/HPOlib/Experiment.py @@ -20,6 +20,7 @@ import logging import os import scipy +from scipy.stats.distributions import wrapcauchy_gen import sys import tempfile @@ -180,8 +181,7 @@ def get_arg_best(self): if np.isfinite(trial['result']): tmp_res = trial['result'] elif np.isfinite(trial['instance_results']).any(): - tmp_res = np.nansum(trial['instance_results']) / \ - (trial['instance_results'].size - np.count_nonzero(~np.isfinite(trial['instance_results']))) + tmp_res = wrapping_util.nan_mean(trial['instance_results']) # np.nanmean is not available in older numpy versions # tmp_res = scipy.nanmean(trial['instance_results']) else: diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 23eb7518..484f7ce9 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -369,9 +369,7 @@ def enqueue_output(out, queue): dct_helper = trials.trials[current_best] res = dct_helper["result"] if \ np.isfinite(dct_helper["result"]) \ - else sum([ele for ele in dct_helper["instance_results"] if np.isfinite(ele)]) / \ - (dct_helper["instance_results"].size - - np.count_nonzero(~np.isfinite(dct_helper["instance_results"]))) + else wrapping_util.nan_mean(dct_helper["instance_results"]) #np.nanmean(trials.trials[current_best]["instance_results"]) # nanmean does not work for all numpy version logger.info("Result %10f, current best %10f", diff --git a/HPOlib/wrapping_util.py b/HPOlib/wrapping_util.py index f7336628..d848af33 100644 --- a/HPOlib/wrapping_util.py +++ b/HPOlib/wrapping_util.py @@ -22,6 +22,7 @@ import logging import imp import math +import numpy as np import traceback import os from StringIO import StringIO @@ -36,6 +37,20 @@ logger = logging.getLogger("HPOlib.wrapping_util") +def nan_mean(arr): + # First: Sum all finite elements + arr = np.array(arr) + res = sum([ele for ele in arr if np.isfinite(ele)]) + num_ele = (arr.size - np.count_nonzero(~np.isfinite(arr))) + if num_ele == 0: + return np.nan + if num_ele != 0 and res == 0: + return 0 + # Second: divide with number of finite elements + res /= num_ele + return res + + def get_time_string(): local_time = datetime.datetime.today() time_string = "%d-%d-%d--%d-%d-%d-%d" % (local_time.year, local_time.month, diff --git a/tests/unittests/test_wrapping_util.py b/tests/unittests/test_wrapping_util.py index 61bb28d4..3333ecfb 100644 --- a/tests/unittests/test_wrapping_util.py +++ b/tests/unittests/test_wrapping_util.py @@ -17,6 +17,7 @@ # along with this program. If not, see . import ConfigParser +import numpy as np import os import shutil import sys @@ -63,7 +64,7 @@ def test_save_config_to_file_ignore_none(self): config.set("HPOLIB", "total_time_limit", None) string_stream = StringIO.StringIO() wrapping_util.save_config_to_file(string_stream, config, - write_nones=False) + write_nones=False) file_content = string_stream.getvalue() asserted_file_content = "[HPOLIB]\n" \ "number_of_jobs = 1\n" \ @@ -85,3 +86,16 @@ def test_use_option_parser_with_config(self): config_args = wrapping_util.parse_config_values_from_unknown_arguments( unknown, config) self.assertEqual(vars(config_args)['HPOLIB:number_of_jobs'], '2') + + def test_nan_mean(self): + self.assertEqual(wrapping_util.nan_mean(np.array([1, 5])), 3) + self.assertEqual(wrapping_util.nan_mean((1, 5)), 3) + # self.assertEqual(wrapping_util.nan_mean("abc"), 3) + self.assertRaises(TypeError, wrapping_util.nan_mean, ("abc")) + self.assertEqual(wrapping_util.nan_mean(np.array([1, 5, np.nan])), 3) + self.assertEqual(wrapping_util.nan_mean(np.array([1, 5, np.inf])), 3) + self.assertTrue(np.isnan(wrapping_util.nan_mean(np.array([np.inf])))) + self.assertEqual(wrapping_util.nan_mean(np.array([-1, 1])), 0) + self.assertTrue(np.isnan(wrapping_util.nan_mean(np.array([])))) + +unittest.main() \ No newline at end of file From 32f347ed54dd7a1bb165de769b11d5148c27fbb6 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Fri, 21 Mar 2014 13:12:58 +0100 Subject: [PATCH 117/139] ENHANCEMENT: Calling HPOlib without config.cfg Calling HPOlib is now possible in a directory without a config.cfg: python ../../HPOlib/wrapping.py -o ../../optimizers/smac/s --SMAC:p=smac_2_06_01-dev/params.txt --HPOLIB:number_of_jobs=20 --HPOLIB:function='python ../branin.py' -s 24 --- HPOlib/check_before_start.py | 4 ++-- HPOlib/config_parser/generalDefault.cfg | 7 ++++++- HPOlib/config_parser/parse.py | 7 +++++-- HPOlib/wrapping_util.py | 15 ++++++++++----- tests/unittests/test_wrapping_util.py | 3 ++- 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index 113c3f1e..2f45a8dd 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -62,7 +62,7 @@ def _check_modules(): try: import theano - logger.debug("\tTheano: %d" % str(theano.__version__)) + logger.debug("\tTheano: %s" % str(theano.__version__)) except ImportError: logger.warning("Theano not found. You might need this to run some " "more complex benchmarks!") @@ -75,7 +75,7 @@ def _check_config(experiment_dir): # check whether config file exists config_file = os.path.join(experiment_dir, "config.cfg") if not os.path.exists(config_file): - raise Exception("There is no config.cfg in %s" % experiment_dir) + logging.warn("There is no config.cfg in %s, all options need to be provided by CLI arguments" % experiment_dir) def check_optimizer(optimizer): diff --git a/HPOlib/config_parser/generalDefault.cfg b/HPOlib/config_parser/generalDefault.cfg index 86a48194..511fe2e9 100644 --- a/HPOlib/config_parser/generalDefault.cfg +++ b/HPOlib/config_parser/generalDefault.cfg @@ -24,4 +24,9 @@ leading_algo_info = # If an algorithm can handle cv on it's own # and calls runsolver_wrapper instead of cv.py as a function -handles_cv=0 \ No newline at end of file +handles_cv=0 + +# Dummy options, which need to be filled out later +number_of_jobs= +function= +result_on_terminate= \ No newline at end of file diff --git a/HPOlib/config_parser/parse.py b/HPOlib/config_parser/parse.py index 588542f7..e468d34b 100644 --- a/HPOlib/config_parser/parse.py +++ b/HPOlib/config_parser/parse.py @@ -37,10 +37,14 @@ def parse_config(config_files, allow_no_value=True, optimizer_version="", os.getcwd(), config_file)) config = ConfigParser.SafeConfigParser(allow_no_value=allow_no_value) + config.read(config_files) if cli_values is not None: config.readfp(cli_values) + return config + +def check_config(config): # -------------------------------------------------------------------------- # Check critical values # -------------------------------------------------------------------------- @@ -53,5 +57,4 @@ def parse_config(config_files, allow_no_value=True, optimizer_version="", if not config.has_option('HPOLIB', 'function') or \ config.get('HPOLIB', 'function') == '': raise Exception('No function specified in .cfg') - - return config + return True diff --git a/HPOlib/wrapping_util.py b/HPOlib/wrapping_util.py index d848af33..697708fb 100644 --- a/HPOlib/wrapping_util.py +++ b/HPOlib/wrapping_util.py @@ -156,16 +156,18 @@ def get_configuration(experiment_dir, optimizer_version, unknown_arguments): sys.exit(1) optimizer_config_fn = os.path.splitext(optimizer_module_parser - .__file__)[0][ - :-7] + "Default.cfg" + .__file__)[0][:-7] + "Default.cfg" if not os.path.exists(optimizer_config_fn): logger.critical("No default config %s found", optimizer_config_fn) sys.exit(1) config_files.append(optimizer_config_fn) - config_files.append(os.path.join(experiment_dir, "config.cfg")) - # Is the config file really the right place to get the allowed keys for a - # hyperparameter optimizer? + path_to_current_config = os.path.join(experiment_dir, "config.cfg") + if os.path.exists(path_to_current_config): + config_files.append(os.path.join(experiment_dir, "config.cfg")) + else: + logger.info("No config file found. Only considering CLI arguments.") + #TODO Is the config file really the right place to get the allowed keys for a hyperparameter optimizer? config = parse.parse_config(config_files, allow_no_value=True, optimizer_version=optimizer_version) @@ -189,6 +191,9 @@ def get_configuration(experiment_dir, optimizer_version, unknown_arguments): fh.close() if optimizer_version != "" and optimizer_version is not None: config = optimizer_module_parser.manipulate_config(config) + + # Check whether we have all necessary options + parse.check_config(config) return config diff --git a/tests/unittests/test_wrapping_util.py b/tests/unittests/test_wrapping_util.py index 3333ecfb..2d927149 100644 --- a/tests/unittests/test_wrapping_util.py +++ b/tests/unittests/test_wrapping_util.py @@ -98,4 +98,5 @@ def test_nan_mean(self): self.assertEqual(wrapping_util.nan_mean(np.array([-1, 1])), 0) self.assertTrue(np.isnan(wrapping_util.nan_mean(np.array([])))) -unittest.main() \ No newline at end of file +if __name__ == "__main__": + unittest.main() From 63564ec5c89ba36117140ded493f3bb31185db51 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 21 Mar 2014 14:07:37 +0100 Subject: [PATCH 118/139] FIX: HPOlib doesn't stop with -q or -v --- HPOlib/wrapping.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index aacd6a5c..92549214 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -378,6 +378,10 @@ def enqueue_output(out, queue): if proc.poll() is not None: # the end condition for the minimal_runs_to_go -= 1 # do-while loop + elif args.verbose or args.silent: + if proc.poll() is not None: + minimal_runs_to_go -= 1 + ret = proc.returncode logger.info("-----------------------END--------------------------------------") From 769c6c5cd6eee97ab4ee8a7f59a5594d20e28f74 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Fri, 21 Mar 2014 14:26:56 +0100 Subject: [PATCH 119/139] REFACTOR: completed config setup for branin --- benchmarks/branin/config.cfg | 8 +++++++- .../branin/smac_2_06_01-dev/{params.txt => params.pcs} | 0 2 files changed, 7 insertions(+), 1 deletion(-) rename benchmarks/branin/smac_2_06_01-dev/{params.txt => params.pcs} (100%) diff --git a/benchmarks/branin/config.cfg b/benchmarks/branin/config.cfg index 20efec78..4a7dbc05 100644 --- a/benchmarks/branin/config.cfg +++ b/benchmarks/branin/config.cfg @@ -1,5 +1,11 @@ [SMAC] -p = smac_2_06_01-dev/params.txt +p = smac_2_06_01-dev/params.pcs + +[TPE] +space = space.py + +[SPEARMINT] +config = config.pb [HPOLIB] function = python ../branin.py diff --git a/benchmarks/branin/smac_2_06_01-dev/params.txt b/benchmarks/branin/smac_2_06_01-dev/params.pcs similarity index 100% rename from benchmarks/branin/smac_2_06_01-dev/params.txt rename to benchmarks/branin/smac_2_06_01-dev/params.pcs From 23f5c93ea6e22f5f33ed3bee4ffe0619e11d768a Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Fri, 21 Mar 2014 14:29:39 +0100 Subject: [PATCH 120/139] REFACTOR: #39 adding lda_on_grid benchmark benchmark folder now contains lda_on_grid benchmark and config files --- HPOlib/benchmark_functions.py | 356 +++++++++++++++++- HPOlib/check_before_start.py | 2 +- benchmarks/lda_ongrid/config.cfg | 13 + .../hyperopt_august2013_mod/space.py | 5 + benchmarks/lda_ongrid/lda.py | 44 +++ .../lda_ongrid/random_hyperopt_august2014_mod | 1 + .../lda_ongrid/smac_2_06_01-dev/params.pcs | 4 + .../spearmint_april2013_mod/config.pb | 27 ++ 8 files changed, 438 insertions(+), 14 deletions(-) create mode 100644 benchmarks/lda_ongrid/config.cfg create mode 100644 benchmarks/lda_ongrid/hyperopt_august2013_mod/space.py create mode 100644 benchmarks/lda_ongrid/lda.py create mode 120000 benchmarks/lda_ongrid/random_hyperopt_august2014_mod create mode 100644 benchmarks/lda_ongrid/smac_2_06_01-dev/params.pcs create mode 100644 benchmarks/lda_ongrid/spearmint_april2013_mod/config.pb diff --git a/HPOlib/benchmark_functions.py b/HPOlib/benchmark_functions.py index 675e04a0..91ce5dbe 100644 --- a/HPOlib/benchmark_functions.py +++ b/HPOlib/benchmark_functions.py @@ -16,7 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from numpy import pi, cos, ndarray, array, exp, NaN +import numpy as np import sys @@ -25,7 +25,7 @@ def save_branin(params, **kwargs): raise ValueError("No params found ['x', 'y']\n") x = float(params["x"]) y = float(params["y"]) - if type(x) == ndarray or type(y) == ndarray: + if type(x) == np.ndarray or type(y) == np.ndarray: x = x[0] y = y[0] @@ -45,23 +45,23 @@ def branin(x, y): three global optima: (-pi, 12.275), (pi, 2.275), (9.42478, 2.475), where branin = 0.397887""" - result = (y-(5.1/(4*pi**2))*x**2+5*x/pi-6)**2 - result += 10*(1-1/(8*pi))*cos(x)+10 + result = (y-(5.1/(4*np.pi**2))*x**2+5*x/np.pi-6)**2 + result += 10*(1-1/(8*np.pi))*np.cos(x)+10 return result def save_har6(params, **kwargs): if "x" not in params or "y" not in params or "z" not in params \ - or "a" not in params or "b" not in params or "c" not in params: + or "a" not in params or "b" not in params or "c" not in params: sys.stderr.write("No params found ['x', 'y']\n") - return NaN + return np.NaN x = float(params["x"]) y = float(params["y"]) z = float(params["z"]) a = float(params["a"]) b = float(params["b"]) c = float(params["c"]) - if type(x) == ndarray: + if type(x) == np.ndarray: x = x[0] y = y[0] z = z[0] @@ -79,7 +79,7 @@ def har6(x, y, z, a, b, c): global optimum at (0.20169, 0.150011, 0.476874, 0.275332, 0.311652, 0.6573), where har6 = -3.32236""" - value = array([x, y, z, a, b, c]) + value = np.array([x, y, z, a, b, c]) if 0 > x or x > 1 or 0 > y or y > 1 or 0 > z or z > 1: raise ValueError("x=%s, y=%s or z=%s not in between 0 and 1" % @@ -88,12 +88,12 @@ def har6(x, y, z, a, b, c): raise ValueError("a=%s, b=%s or c=%s not in between 0 and 1" % (a, b, c)) - a = array([[10.0, 3.0, 17.0, 3.5, 1.7, 8.0], + a = np.array([[10.0, 3.0, 17.0, 3.5, 1.7, 8.0], [ 0.05, 10.0, 17.0, 0.1, 8.0, 14.0], [ 3.0, 3.5, 1.7, 10.0, 17.0, 8.0], [17.0, 8.0, 0.05, 10.0, 0.1, 14.0]]) - c = array([1.0, 1.2, 3.0, 3.2]) - p = array([[0.1312, 0.1696, 0.5569, 0.0124, 0.8283, 0.5886], + c = np.array([1.0, 1.2, 3.0, 3.2]) + p = np.array([[0.1312, 0.1696, 0.5569, 0.0124, 0.8283, 0.5886], [0.2329, 0.4135, 0.8307, 0.3736, 0.1004, 0.9991], [0.2348, 0.1451, 0.3522, 0.2883, 0.3047, 0.6650], [0.4047, 0.8828, 0.8732, 0.5743, 0.1091, 0.0381]]) @@ -105,6 +105,336 @@ def har6(x, y, z, a, b, c): sm += a[i,3]*(value[3]-p[i,3])**2 sm += a[i,4]*(value[4]-p[i,4])**2 sm += a[i,5]*(value[5]-p[i,5])**2 - s += c[i]*exp(-sm) + s += c[i]*np.exp(-sm) result = -s - return result \ No newline at end of file + return result + + +def save_lda_on_grid(params, **kwargs): + if "Kappa" not in params or "Tau" not in params or "S" not in params: + sys.stderr.write("No params found ['Kappa', 'Tau', 'S']: %s\n" % str(params)) + return np.NaN + kappa = int(float(params["Kappa"])) + tau = int(float(params["Tau"])) + s = int(float(params["S"])) + + if type(kappa) == np.ndarray: + kappa = kappa[0] + tau = tau[0] + s = s[0] + return lda_on_grid(kappa, tau, s) + + +def lda_on_grid(kappa, tau, s): + # Values for an 6*8*8 grid search which was performed by Hofman et. al. + # Values obtained from Jasper Snoek + # Kappa Tau S Value Time + configurations = np.array([[1.000000, 4.000000, 16.000000, 2014.255351, 36393.190000], + [0.900000, 1024.000000, 4096.000000, 1680.540179, 36419.510000], + [0.600000, 1024.000000, 4096.000000, 1328.191297, 24219.850000], + [0.600000, 16.000000, 4.000000, 2859.440420, 18862.010000], + [0.700000, 1.000000, 4096.000000, 1313.263743, 11218.420000], + [0.500000, 64.000000, 16384.000000, 1271.869567, 18228.940000], + [0.600000, 64.000000, 4.000000, 2630.424468, 18970.510000], + [1.000000, 1.000000, 1.000000, 2547.784743, 36317.510000], + [1.000000, 16.000000, 64.000000, 1558.076377, 29985.160000], + [0.500000, 256.000000, 16384.000000, 1278.123799, 20822.350000], + [0.800000, 256.000000, 4096.000000, 1361.320422, 30758.540000], + [0.600000, 256.000000, 16.000000, 1800.429160, 10775.890000], + [0.500000, 4.000000, 64.000000, 2084.897678, 14864.830000], + [0.800000, 64.000000, 256.000000, 1349.812010, 17315.100000], + [1.000000, 16.000000, 4.000000, 2079.802409, 36338.350000], + [0.500000, 1.000000, 4.000000, 3624.167804, 12595.790000], + [0.900000, 16.000000, 16.000000, 1749.505296, 27772.590000], + [0.800000, 1.000000, 4096.000000, 1310.060860, 15432.120000], + [1.000000, 64.000000, 1024.000000, 1343.801912, 19662.620000], + [0.700000, 1.000000, 64.000000, 1983.587762, 10802.270000], + [0.500000, 4.000000, 16.000000, 2804.748620, 12678.520000], + [0.700000, 256.000000, 64.000000, 1427.856381, 10884.970000], + [0.900000, 16.000000, 1024.000000, 1317.263697, 15282.860000], + [0.500000, 256.000000, 1.000000, 5000.332116, 12602.320000], + [1.000000, 1024.000000, 4.000000, 1637.661866, 36225.830000], + [0.800000, 1024.000000, 4096.000000, 1537.873629, 36424.780000], + [0.900000, 16.000000, 4.000000, 2040.531678, 36327.940000], + [0.500000, 64.000000, 1.000000, 5022.511473, 14693.930000], + [1.000000, 4.000000, 256.000000, 1472.690781, 25890.240000], + [1.000000, 64.000000, 16.000000, 1572.944521, 36307.320000], + [0.800000, 1024.000000, 4.000000, 1528.555961, 19240.020000], + [0.800000, 64.000000, 16.000000, 1631.072354, 15148.720000], + [0.900000, 64.000000, 16.000000, 1541.968527, 25795.950000], + [0.900000, 1024.000000, 256.000000, 1468.342464, 25968.680000], + [0.700000, 4.000000, 1.000000, 2968.382913, 27098.820000], + [0.900000, 1.000000, 64.000000, 1886.883404, 23495.740000], + [0.800000, 4.000000, 16.000000, 2101.979941, 14956.600000], + [0.600000, 1.000000, 16.000000, 2601.345619, 10651.960000], + [0.900000, 1024.000000, 64.000000, 1506.415675, 21523.900000], + [0.700000, 16.000000, 16384.000000, 1271.574590, 24894.000000], + [1.000000, 1.000000, 4.000000, 2524.250165, 36327.700000], + [1.000000, 1024.000000, 16384.000000, 2250.711024, 36354.110000], + [0.800000, 4.000000, 1.000000, 2566.382537, 29395.750000], + [0.600000, 64.000000, 64.000000, 1630.248431, 8734.110000], + [0.800000, 16.000000, 16.000000, 1836.733098, 17066.550000], + [0.500000, 16.000000, 16384.000000, 1266.167382, 16119.520000], + [0.500000, 1024.000000, 256.000000, 1364.836274, 10978.410000], + [0.600000, 1.000000, 1024.000000, 1435.221761, 8879.080000], + [0.600000, 4.000000, 4.000000, 3053.489799, 18842.800000], + [0.600000, 64.000000, 16384.000000, 1268.070557, 22757.820000], + [1.000000, 4.000000, 16384.000000, 1293.198126, 27337.720000], + [1.000000, 4.000000, 64.000000, 1718.817531, 30082.460000], + [0.600000, 1024.000000, 4.000000, 2045.705842, 14828.510000], + [0.500000, 1.000000, 256.000000, 1730.222585, 8710.590000], + [0.700000, 1024.000000, 1024.000000, 1369.038389, 19597.910000], + [0.900000, 64.000000, 64.000000, 1454.543818, 23658.300000], + [0.600000, 256.000000, 256.000000, 1353.926401, 10924.180000], + [0.900000, 4.000000, 4.000000, 2240.094436, 35811.290000], + [0.900000, 1024.000000, 1024.000000, 1521.803360, 36370.860000], + [0.900000, 16.000000, 64.000000, 1537.363365, 25713.840000], + [1.000000, 1024.000000, 256.000000, 1612.244480, 34376.890000], + [0.600000, 1024.000000, 1.000000, 2918.561304, 12676.230000], + [0.800000, 1024.000000, 64.000000, 1447.394204, 19376.390000], + [1.000000, 16.000000, 4096.000000, 1295.767130, 19752.540000], + [0.900000, 16.000000, 4096.000000, 1284.176859, 17665.830000], + [0.500000, 64.000000, 64.000000, 1786.323249, 17090.440000], + [0.800000, 1.000000, 256.000000, 1590.643012, 13020.470000], + [0.600000, 1.000000, 1.000000, 3722.019016, 12581.950000], + [0.500000, 64.000000, 16.000000, 2346.008412, 10628.190000], + [0.700000, 16.000000, 256.000000, 1419.439444, 10909.080000], + [1.000000, 64.000000, 64.000000, 1495.508178, 27974.310000], + [0.900000, 256.000000, 16384.000000, 1568.119023, 36540.200000], + [0.700000, 256.000000, 1024.000000, 1309.209494, 13202.940000], + [1.000000, 256.000000, 16.000000, 1593.476294, 29959.260000], + [0.600000, 16.000000, 16384.000000, 1267.214011, 18165.470000], + [0.700000, 1024.000000, 4.000000, 1711.022236, 12862.920000], + [1.000000, 64.000000, 4096.000000, 1356.962017, 28415.150000], + [0.800000, 16.000000, 4.000000, 2175.249552, 21110.880000], + [0.900000, 4.000000, 64.000000, 1706.756992, 23604.100000], + [1.000000, 256.000000, 1024.000000, 1450.749101, 32559.760000], + [1.000000, 1024.000000, 16.000000, 1696.944579, 25716.020000], + [0.700000, 1024.000000, 64.000000, 1413.755921, 10896.830000], + [0.700000, 4.000000, 16384.000000, 1273.726730, 18109.140000], + [0.600000, 4.000000, 4096.000000, 1299.255183, 11151.040000], + [0.600000, 4.000000, 1024.000000, 1382.858640, 8897.740000], + [0.900000, 1.000000, 1.000000, 2364.737107, 36310.600000], + [0.900000, 64.000000, 16384.000000, 1383.792566, 36559.280000], + [0.800000, 1.000000, 16.000000, 2271.844459, 16982.230000], + [0.700000, 256.000000, 1.000000, 2566.918453, 16962.530000], + [0.900000, 256.000000, 1.000000, 1865.777546, 36325.050000], + [0.800000, 64.000000, 4096.000000, 1296.580970, 19881.550000], + [0.900000, 256.000000, 1024.000000, 1392.699949, 26037.380000], + [0.700000, 64.000000, 64.000000, 1505.616635, 10928.860000], + [0.900000, 1.000000, 16384.000000, 1287.551166, 23238.140000], + [0.600000, 16.000000, 4096.000000, 1285.295193, 11148.520000], + [0.500000, 1.000000, 4096.000000, 1329.484556, 8947.940000], + [1.000000, 16.000000, 1.000000, 2369.474345, 36322.510000], + [0.900000, 256.000000, 256.000000, 1391.712439, 17420.020000], + [0.800000, 1024.000000, 1.000000, 1787.887829, 29570.520000], + [0.900000, 64.000000, 256.000000, 1358.717854, 17338.630000], + [0.800000, 4.000000, 16384.000000, 1273.680399, 22885.930000], + [0.800000, 16.000000, 16384.000000, 1277.089752, 24991.040000], + [0.600000, 1.000000, 4096.000000, 1317.818664, 11117.430000], + [0.800000, 4.000000, 1024.000000, 1343.234780, 15387.070000], + [0.500000, 256.000000, 4096.000000, 1274.208686, 11165.100000], + [0.800000, 1024.000000, 1024.000000, 1420.742784, 30340.100000], + [0.500000, 1024.000000, 16384.000000, 1321.591240, 33868.750000], + [0.700000, 1024.000000, 1.000000, 2204.569258, 29469.740000], + [0.700000, 1024.000000, 16384.000000, 1524.871729, 36434.350000], + [0.600000, 1024.000000, 1024.000000, 1317.814555, 13172.090000], + [0.700000, 64.000000, 256.000000, 1359.437760, 13076.980000], + [0.900000, 4.000000, 16.000000, 2006.496131, 29844.430000], + [0.800000, 1.000000, 4.000000, 2477.453345, 23270.780000], + [0.500000, 1024.000000, 64.000000, 1523.244259, 8761.100000], + [0.800000, 256.000000, 1.000000, 2077.487352, 29570.100000], + [0.600000, 16.000000, 1024.000000, 1334.947039, 11038.730000], + [1.000000, 256.000000, 16384.000000, 1697.459511, 36489.610000], + [0.800000, 256.000000, 16.000000, 1504.079040, 15143.090000], + [0.500000, 4.000000, 1024.000000, 1413.822886, 10958.240000], + [0.500000, 64.000000, 4096.000000, 1278.813005, 11103.810000], + [0.500000, 1.000000, 64.000000, 2227.201820, 10742.500000], + [1.000000, 1.000000, 16384.000000, 1301.175618, 25068.840000], + [0.500000, 4.000000, 256.000000, 1643.681520, 10869.120000], + [0.800000, 16.000000, 1.000000, 2560.527040, 29376.070000], + [0.500000, 16.000000, 1024.000000, 1364.872251, 8879.340000], + [0.800000, 16.000000, 64.000000, 1572.391232, 15021.860000], + [1.000000, 1024.000000, 1024.000000, 1738.719119, 36385.890000], + [0.800000, 1024.000000, 16.000000, 1509.744747, 17226.820000], + [0.900000, 1024.000000, 1.000000, 1654.893494, 36341.060000], + [1.000000, 16.000000, 1024.000000, 1335.264417, 15416.250000], + [0.900000, 256.000000, 64.000000, 1453.060437, 21542.420000], + [0.700000, 16.000000, 64.000000, 1668.761813, 10853.180000], + [0.800000, 16.000000, 1024.000000, 1309.582372, 15377.860000], + [0.600000, 256.000000, 1.000000, 3294.021457, 12682.510000], + [1.000000, 1024.000000, 4096.000000, 1847.090224, 36369.090000], + [0.700000, 64.000000, 16.000000, 1784.594764, 10797.140000], + [0.900000, 1.000000, 256.000000, 1582.721013, 17274.740000], + [0.700000, 256.000000, 256.000000, 1343.241802, 13073.810000], + [0.500000, 64.000000, 256.000000, 1457.530339, 10908.290000], + [0.600000, 1024.000000, 64.000000, 1438.727854, 8780.440000], + [0.600000, 256.000000, 4.000000, 2360.980123, 18943.070000], + [0.600000, 4.000000, 64.000000, 1943.349721, 10784.510000], + [0.900000, 16.000000, 16384.000000, 1282.164335, 27758.470000], + [1.000000, 4.000000, 4.000000, 2306.935465, 36328.220000], + [0.500000, 256.000000, 16.000000, 2088.040327, 10697.110000], + [0.600000, 4.000000, 16.000000, 2482.891469, 10661.270000], + [0.700000, 64.000000, 1.000000, 2810.881943, 27147.190000], + [0.700000, 4.000000, 16.000000, 2259.310536, 12780.130000], + [0.900000, 256.000000, 4.000000, 1601.761693, 36219.240000], + [0.500000, 64.000000, 4.000000, 3198.956320, 12589.520000], + [0.800000, 1.000000, 64.000000, 1917.207978, 14986.340000], + [0.600000, 1.000000, 16384.000000, 1272.839987, 16139.940000], + [0.500000, 16.000000, 4.000000, 3346.584734, 22882.000000], + [0.900000, 1024.000000, 4.000000, 1513.696030, 31977.800000], + [1.000000, 4.000000, 1024.000000, 1358.437431, 19614.580000], + [1.000000, 1.000000, 16.000000, 2244.860585, 36330.680000], + [1.000000, 256.000000, 1.000000, 1855.132826, 36336.080000], + [0.900000, 256.000000, 16.000000, 1509.932842, 27889.120000], + [1.000000, 16.000000, 16384.000000, 1302.230813, 27616.620000], + [0.500000, 4.000000, 1.000000, 5149.513805, 12552.220000], + [1.000000, 64.000000, 1.000000, 2109.894998, 36349.640000], + [0.600000, 64.000000, 1.000000, 3619.232213, 12633.430000], + [0.800000, 256.000000, 4.000000, 1713.158518, 23433.050000], + [0.900000, 64.000000, 1024.000000, 1318.388432, 17534.690000], + [0.700000, 256.000000, 16.000000, 1594.416105, 10857.250000], + [0.600000, 1.000000, 4.000000, 3126.891155, 18807.430000], + [0.600000, 4.000000, 16384.000000, 1270.425851, 16183.590000], + [0.800000, 16.000000, 256.000000, 1385.504627, 17284.460000], + [1.000000, 64.000000, 256.000000, 1388.593292, 19561.620000], + [0.800000, 256.000000, 256.000000, 1347.624790, 17293.180000], + [0.600000, 16.000000, 64.000000, 1786.518680, 10804.840000], + [0.900000, 64.000000, 4096.000000, 1318.135676, 21929.520000], + [0.500000, 1.000000, 1.000000, 5258.112826, 12536.720000], + [0.600000, 1.000000, 256.000000, 1660.049798, 8731.610000], + [0.700000, 64.000000, 1024.000000, 1303.411361, 13189.740000], + [0.900000, 1.000000, 16.000000, 2210.134257, 29606.270000], + [0.500000, 1024.000000, 4096.000000, 1296.897775, 15553.910000], + [0.600000, 4.000000, 1.000000, 3747.871323, 12555.700000], + [0.500000, 16.000000, 1.000000, 5229.181160, 12538.960000], + [0.500000, 1024.000000, 16.000000, 1880.399470, 10733.760000], + [0.700000, 256.000000, 4096.000000, 1314.937940, 19874.800000], + [0.900000, 1024.000000, 16.000000, 1586.647560, 21473.500000], + [1.000000, 16.000000, 256.000000, 1404.940141, 21758.270000], + [0.700000, 1.000000, 256.000000, 1619.226174, 10852.560000], + [0.800000, 256.000000, 64.000000, 1408.231244, 19407.500000], + [0.900000, 4.000000, 16384.000000, 1280.796294, 25147.340000], + [0.800000, 256.000000, 1024.000000, 1330.546637, 19667.980000], + [0.500000, 4.000000, 16384.000000, 1270.589627, 13865.620000], + [0.700000, 16.000000, 4096.000000, 1281.538285, 13378.880000], + [0.800000, 64.000000, 16384.000000, 1320.508949, 34243.490000], + [0.900000, 1.000000, 1024.000000, 1405.254246, 17379.910000], + [0.900000, 16.000000, 1.000000, 2341.462081, 36329.320000], + [0.700000, 64.000000, 16384.000000, 1282.492457, 27080.430000], + [0.700000, 256.000000, 4.000000, 1951.044824, 12853.060000], + [1.000000, 1.000000, 256.000000, 1593.512913, 21558.680000], + [1.000000, 256.000000, 4096.000000, 1588.528832, 36406.420000], + [0.900000, 1.000000, 4.000000, 2429.684277, 33713.730000], + [0.500000, 1.000000, 16384.000000, 1271.753365, 13760.600000], + [1.000000, 1024.000000, 64.000000, 1579.061710, 25761.850000], + [0.700000, 16.000000, 4.000000, 2487.987431, 18947.020000], + [0.500000, 4.000000, 4096.000000, 1309.276414, 8971.530000], + [0.800000, 4.000000, 4096.000000, 1289.574825, 15376.110000], + [0.900000, 4.000000, 4096.000000, 1294.339616, 17704.140000], + [0.500000, 1024.000000, 1024.000000, 1302.023694, 10982.080000], + [0.600000, 1024.000000, 256.000000, 1351.998671, 10947.490000], + [0.500000, 1024.000000, 4.000000, 2542.238814, 29202.250000], + [0.800000, 4.000000, 256.000000, 1478.045732, 15249.950000], + [1.000000, 4.000000, 4096.000000, 1304.324411, 19810.090000], + [0.900000, 64.000000, 1.000000, 2118.723891, 36320.250000], + [0.600000, 64.000000, 16.000000, 2033.429989, 10707.310000], + [1.000000, 1.000000, 64.000000, 1913.104369, 27754.330000], + [0.500000, 256.000000, 1024.000000, 1307.561280, 11039.350000], + [0.600000, 1024.000000, 16384.000000, 1393.614637, 36791.210000], + [1.000000, 16.000000, 16.000000, 1750.590736, 36267.290000], + [0.500000, 256.000000, 64.000000, 1626.654117, 14953.820000], + [0.900000, 256.000000, 4096.000000, 1439.558928, 36385.390000], + [0.800000, 1.000000, 1.000000, 2548.705258, 29322.290000], + [0.700000, 1024.000000, 256.000000, 1363.001016, 15230.860000], + [0.900000, 4.000000, 256.000000, 1463.333299, 19454.290000], + [0.800000, 1.000000, 16384.000000, 1281.177353, 20554.140000], + [0.600000, 1.000000, 64.000000, 2086.676125, 10750.130000], + [1.000000, 1.000000, 4096.000000, 1323.445870, 21980.400000], + [0.700000, 4.000000, 4.000000, 2711.405961, 12692.900000], + [0.600000, 256.000000, 1024.000000, 1302.341384, 11066.980000], + [0.700000, 1.000000, 16384.000000, 1276.738560, 18358.530000], + [0.600000, 256.000000, 64.000000, 1501.526137, 8766.400000], + [0.700000, 4.000000, 64.000000, 1843.379712, 10832.460000], + [0.500000, 256.000000, 256.000000, 1401.072001, 6741.450000], + [0.700000, 1.000000, 4.000000, 2729.267232, 23222.120000], + [0.800000, 4.000000, 4.000000, 2384.413805, 23341.310000], + [0.600000, 256.000000, 16384.000000, 1310.790773, 32127.290000], + [0.600000, 64.000000, 1024.000000, 1309.290893, 10988.550000], + [0.600000, 4.000000, 256.000000, 1568.538127, 10911.060000], + [0.700000, 64.000000, 4096.000000, 1276.161920, 15525.330000], + [0.700000, 1024.000000, 4096.000000, 1397.052266, 36423.470000], + [0.800000, 16.000000, 4096.000000, 1280.470261, 15551.190000], + [0.800000, 1.000000, 1024.000000, 1406.231488, 13228.600000], + [0.600000, 16.000000, 256.000000, 1473.258879, 10886.050000], + [0.800000, 64.000000, 1024.000000, 1304.882655, 15346.050000], + [0.700000, 4.000000, 4096.000000, 1291.760503, 13324.520000], + [0.700000, 1.000000, 16.000000, 2422.500695, 14833.800000], + [0.800000, 64.000000, 1.000000, 2347.846308, 29466.530000], + [0.900000, 4.000000, 1.000000, 2452.668607, 36313.550000], + [0.500000, 1024.000000, 64.000000, 1520.064450, 10877.800000], + [0.800000, 1024.000000, 256.000000, 1389.175687, 17386.800000], + [0.700000, 256.000000, 16384.000000, 1377.931876, 36591.050000], + [0.500000, 1024.000000, 1.000000, 4432.511510, 12621.590000], + [0.900000, 16.000000, 256.000000, 1381.783824, 21799.460000], + [0.900000, 4.000000, 1024.000000, 1345.175995, 17429.470000], + [0.600000, 1024.000000, 16.000000, 1626.743743, 12900.210000], + [0.900000, 64.000000, 4.000000, 1789.856352, 31967.840000], + [0.600000, 256.000000, 4096.000000, 1290.394505, 17664.880000], + [1.000000, 256.000000, 64.000000, 1503.488973, 21555.060000], + [1.000000, 4.000000, 1.000000, 2562.137775, 36323.630000], + [1.000000, 1.000000, 1024.000000, 1416.648457, 19595.790000], + [0.700000, 1.000000, 1.000000, 2975.404068, 16789.480000], + [0.500000, 16.000000, 256.000000, 1549.807417, 10858.790000], + [0.700000, 16.000000, 1.000000, 2961.509740, 27084.010000], + [0.900000, 1024.000000, 16384.000000, 1929.981674, 36384.180000], + [1.000000, 1024.000000, 1.000000, 1693.890539, 36343.210000], + [0.900000, 1.000000, 4096.000000, 1314.611328, 17626.750000], + [0.500000, 16.000000, 16.000000, 2612.878741, 10712.800000], + [0.500000, 4.000000, 4.000000, 3481.699285, 29060.450000], + [0.500000, 256.000000, 4.000000, 2939.348266, 12609.960000], + [0.700000, 4.000000, 1024.000000, 1360.104102, 10955.300000], + [0.500000, 1.000000, 1024.000000, 1460.865658, 8810.670000], + [0.800000, 1024.000000, 16384.000000, 1670.162786, 36825.090000], + [0.500000, 16.000000, 64.000000, 1969.256740, 6563.080000], + [0.700000, 1024.000000, 16.000000, 1505.384479, 10848.440000], + [0.500000, 64.000000, 1024.000000, 1329.959506, 8891.580000], + [0.800000, 256.000000, 16384.000000, 1442.360685, 36395.860000], + [0.700000, 16.000000, 16.000000, 2035.164063, 12877.330000], + [0.700000, 16.000000, 1024.000000, 1318.230944, 11043.220000], + [0.800000, 64.000000, 4.000000, 1943.102596, 27575.300000], + [0.700000, 64.000000, 4.000000, 2250.809494, 12793.650000], + [0.600000, 64.000000, 4096.000000, 1271.714717, 13322.590000], + [0.600000, 16.000000, 1.000000, 3770.110326, 12621.170000], + [1.000000, 256.000000, 256.000000, 1417.792032, 21866.210000], + [1.000000, 256.000000, 4.000000, 1620.128845, 36348.480000], + [0.600000, 64.000000, 256.000000, 1395.590972, 10909.370000], + [0.700000, 1.000000, 1024.000000, 1414.752908, 10989.010000], + [1.000000, 64.000000, 4.000000, 1812.145883, 36335.070000], + [0.700000, 4.000000, 256.000000, 1521.322055, 10894.450000], + [0.500000, 16.000000, 4096.000000, 1292.698578, 9022.760000], + [0.500000, 1.000000, 16.000000, 2836.012122, 16765.670000], + [1.000000, 64.000000, 16384.000000, 1433.882412, 36474.950000], + [0.800000, 64.000000, 64.000000, 1454.028740, 17186.180000], + [0.800000, 4.000000, 64.000000, 1752.565517, 15054.510000], + [0.600000, 16.000000, 16.000000, 2277.597145, 10683.480000]]) + + kappa_values = (0.5, 0.6, 0.7, 0.8, 0.9, 1.0) + tau_values = (1.0, 4.0, 16.0, 64.0, 256.0, 1024.0) + s_values = (1.0, 4.0, 16.0, 64.0, 256.0, 1024.0, 4096.0, 16384.0) + + # Convert the array into an easily accessible tree + config_tree = dict() + for kappa_value in kappa_values: + config_tree[kappa_value] = dict() + for Tau_value in tau_values: + config_tree[kappa_value][Tau_value] = dict() + for S_value in s_values: + config_tree[kappa_value][Tau_value][S_value] = np.inf + + # Populate the tree + for config in configurations: + config_tree[config[0]][config[1]][config[2]] = config[3] + + print kappa, tau, s + return config_tree[kappa_values[int(float(kappa))]][tau_values[int(float(tau))]][s_values[int(float(s))]] \ No newline at end of file diff --git a/HPOlib/check_before_start.py b/HPOlib/check_before_start.py index 2f45a8dd..b89319f4 100644 --- a/HPOlib/check_before_start.py +++ b/HPOlib/check_before_start.py @@ -75,7 +75,7 @@ def _check_config(experiment_dir): # check whether config file exists config_file = os.path.join(experiment_dir, "config.cfg") if not os.path.exists(config_file): - logging.warn("There is no config.cfg in %s, all options need to be provided by CLI arguments" % experiment_dir) + logger.warn("There is no config.cfg in %s, all options need to be provided by CLI arguments" % experiment_dir) def check_optimizer(optimizer): diff --git a/benchmarks/lda_ongrid/config.cfg b/benchmarks/lda_ongrid/config.cfg new file mode 100644 index 00000000..6c245bdb --- /dev/null +++ b/benchmarks/lda_ongrid/config.cfg @@ -0,0 +1,13 @@ +[SMAC] +p = smac_2_06_01-dev/params.pcs + +[TPE] +space = space.py + +[SPEARMINT] +config = config.pb + +[HPOLIB] +function = python ../lda.py +number_of_jobs = 50 +result_on_terminate = 1000000000 diff --git a/benchmarks/lda_ongrid/hyperopt_august2013_mod/space.py b/benchmarks/lda_ongrid/hyperopt_august2013_mod/space.py new file mode 100644 index 00000000..3b8ead30 --- /dev/null +++ b/benchmarks/lda_ongrid/hyperopt_august2013_mod/space.py @@ -0,0 +1,5 @@ +from hyperopt import hp + +space = {'Kappa': hp.quniform('Kappa', 0, 5, 1), + 'Tau': hp.quniform('Tau', 0, 5, 1), + 'S': hp.quniform('S', 0, 7, 1)} diff --git a/benchmarks/lda_ongrid/lda.py b/benchmarks/lda_ongrid/lda.py new file mode 100644 index 00000000..41bb6294 --- /dev/null +++ b/benchmarks/lda_ongrid/lda.py @@ -0,0 +1,44 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import time + +import numpy as np + +import HPOlib.benchmark_util as benchmark_util +import HPOlib.benchmark_functions as benchmark_functions + +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" +__credits__ = ["Jasper Snoek", "Ryan P. Adams", "Hugo Larochelle"] + + +def main(params, **kwargs): + print 'Params: ', params + y = benchmark_functions.save_lda_on_grid(params, **kwargs) + print 'Result: ', y + return y + + +if __name__ == "__main__": + starttime = time.time() + args, params = benchmark_util.parse_cli() + result = main(params, **args) + duration = time.time() - starttime + print "Result for ParamILS: %s, %f, 1, %f, %d, %s" % \ + ("SAT", abs(duration), result, -1, str(__file__)) \ No newline at end of file diff --git a/benchmarks/lda_ongrid/random_hyperopt_august2014_mod b/benchmarks/lda_ongrid/random_hyperopt_august2014_mod new file mode 120000 index 00000000..c4f9ce32 --- /dev/null +++ b/benchmarks/lda_ongrid/random_hyperopt_august2014_mod @@ -0,0 +1 @@ +hyperopt_august2013_mod \ No newline at end of file diff --git a/benchmarks/lda_ongrid/smac_2_06_01-dev/params.pcs b/benchmarks/lda_ongrid/smac_2_06_01-dev/params.pcs new file mode 100644 index 00000000..c3551e94 --- /dev/null +++ b/benchmarks/lda_ongrid/smac_2_06_01-dev/params.pcs @@ -0,0 +1,4 @@ +Kappa [0, 5] [0]i +Tau [0, 5] [0]i +S [0, 7] [0]i + diff --git a/benchmarks/lda_ongrid/spearmint_april2013_mod/config.pb b/benchmarks/lda_ongrid/spearmint_april2013_mod/config.pb new file mode 100644 index 00000000..0dff3cdb --- /dev/null +++ b/benchmarks/lda_ongrid/spearmint_april2013_mod/config.pb @@ -0,0 +1,27 @@ +language: PYTHON +name: "HPOlib.cv" + +variable { + name: "Kappa" + type: INT + size: 1 + min: 0 + max: 5 +} + +variable { + name: "Tau" + type: INT + size: 1 + min: 0 + max: 5 +} + +variable { + name: "S" + type: INT + size: 1 + min: 0 + max: 7 +} + From 550b9f74f556246a6ec2e6f49bb21707ee7be8ed Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Fri, 21 Mar 2014 15:45:33 +0100 Subject: [PATCH 121/139] REFACTOR: lda_on_grid returns duration --- benchmarks/lda_ongrid/lda.py | 12 ++++-------- benchmarks/lda_ongrid/random_hyperopt_august2013_mod | 1 + 2 files changed, 5 insertions(+), 8 deletions(-) create mode 120000 benchmarks/lda_ongrid/random_hyperopt_august2013_mod diff --git a/benchmarks/lda_ongrid/lda.py b/benchmarks/lda_ongrid/lda.py index 41bb6294..c21a1125 100644 --- a/benchmarks/lda_ongrid/lda.py +++ b/benchmarks/lda_ongrid/lda.py @@ -16,9 +16,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import time - -import numpy as np import HPOlib.benchmark_util as benchmark_util import HPOlib.benchmark_functions as benchmark_functions @@ -28,17 +25,16 @@ __credits__ = ["Jasper Snoek", "Ryan P. Adams", "Hugo Larochelle"] -def main(params, **kwargs): +def main(params, ret_time=False, **kwargs): print 'Params: ', params - y = benchmark_functions.save_lda_on_grid(params, **kwargs) + y = benchmark_functions.save_lda_on_grid(params, ret_time=ret_time, **kwargs) print 'Result: ', y return y if __name__ == "__main__": - starttime = time.time() args, params = benchmark_util.parse_cli() - result = main(params, **args) - duration = time.time() - starttime + result = main(params, ret_time=False, **args) + duration = main(params, ret_time=True) print "Result for ParamILS: %s, %f, 1, %f, %d, %s" % \ ("SAT", abs(duration), result, -1, str(__file__)) \ No newline at end of file diff --git a/benchmarks/lda_ongrid/random_hyperopt_august2013_mod b/benchmarks/lda_ongrid/random_hyperopt_august2013_mod new file mode 120000 index 00000000..c4f9ce32 --- /dev/null +++ b/benchmarks/lda_ongrid/random_hyperopt_august2013_mod @@ -0,0 +1 @@ +hyperopt_august2013_mod \ No newline at end of file From 0310d4d57961245c27b4c9c209aa8d900fd9544d Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Fri, 21 Mar 2014 15:46:43 +0100 Subject: [PATCH 122/139] REFACTOR: #39 Adding svm_on_grid benchmark --- HPOlib/benchmark_functions.py | 1487 ++++++++++++++++- benchmarks/svm_on_grid/config.cfg | 13 + .../random_hyperopt_august2013_mod | 1 + .../hyperopt_august2013_mod/space.py | 5 + .../random_hyperopt_august2013_mod | 1 + .../svm_on_grid/smac_2_06_01-dev/params.pcs | 4 + .../spearmint_april2013_mod/config.pb | 27 + benchmarks/svm_on_grid/svm.py | 41 + 8 files changed, 1573 insertions(+), 6 deletions(-) create mode 100644 benchmarks/svm_on_grid/config.cfg create mode 120000 benchmarks/svm_on_grid/hyperopt_august2013_mod/random_hyperopt_august2013_mod create mode 100644 benchmarks/svm_on_grid/hyperopt_august2013_mod/space.py create mode 120000 benchmarks/svm_on_grid/random_hyperopt_august2013_mod create mode 100644 benchmarks/svm_on_grid/smac_2_06_01-dev/params.pcs create mode 100644 benchmarks/svm_on_grid/spearmint_april2013_mod/config.pb create mode 100644 benchmarks/svm_on_grid/svm.py diff --git a/HPOlib/benchmark_functions.py b/HPOlib/benchmark_functions.py index 91ce5dbe..c352e984 100644 --- a/HPOlib/benchmark_functions.py +++ b/HPOlib/benchmark_functions.py @@ -110,7 +110,7 @@ def har6(x, y, z, a, b, c): return result -def save_lda_on_grid(params, **kwargs): +def save_lda_on_grid(params, ret_time=False, **kwargs): if "Kappa" not in params or "Tau" not in params or "S" not in params: sys.stderr.write("No params found ['Kappa', 'Tau', 'S']: %s\n" % str(params)) return np.NaN @@ -122,13 +122,14 @@ def save_lda_on_grid(params, **kwargs): kappa = kappa[0] tau = tau[0] s = s[0] - return lda_on_grid(kappa, tau, s) + return lda_on_grid(kappa, tau, s, ret_time=ret_time) -def lda_on_grid(kappa, tau, s): +def lda_on_grid(kappa, tau, s, ret_time=False): # Values for an 6*8*8 grid search which was performed by Hofman et. al. # Values obtained from Jasper Snoek # Kappa Tau S Value Time + # ret_time: return time instead of perplexity configurations = np.array([[1.000000, 4.000000, 16.000000, 2014.255351, 36393.190000], [0.900000, 1024.000000, 4096.000000, 1680.540179, 36419.510000], [0.600000, 1024.000000, 4096.000000, 1328.191297, 24219.850000], @@ -430,11 +431,1485 @@ def lda_on_grid(kappa, tau, s): for Tau_value in tau_values: config_tree[kappa_value][Tau_value] = dict() for S_value in s_values: - config_tree[kappa_value][Tau_value][S_value] = np.inf + config_tree[kappa_value][Tau_value][S_value] = [np.inf, np.inf] # Populate the tree for config in configurations: - config_tree[config[0]][config[1]][config[2]] = config[3] + config_tree[config[0]][config[1]][config[2]][0] = config[3] + config_tree[config[0]][config[1]][config[2]][1] = config[4] print kappa, tau, s - return config_tree[kappa_values[int(float(kappa))]][tau_values[int(float(tau))]][s_values[int(float(s))]] \ No newline at end of file + if ret_time: + return config_tree[kappa_values[kappa]][tau_values[tau]][s_values[s]][1] + else: + return config_tree[kappa_values[kappa]][tau_values[tau]][s_values[s]][1] + + +def save_svm_on_grid(params, ret_time=False, **kwargs): + if "C" not in params or "alpha" not in params or "epsilon" not in params: + sys.stderr.write("No params found ['C', 'alpha', 'epsilon']: %s\n" % str(params)) + return np.NaN + c = int(float(params["C"])) + alpha = int(float(params["alpha"])) + epsilon = int(float(params["epsilon"])) + + if type(c) == np.ndarray: + c = c[0] + alpha = alpha[0] + epsilon = epsilon[0] + return svm_on_grid(c=c, alpha=alpha, epsilon=epsilon, ret_time=ret_time) + + +def svm_on_grid(c, alpha, epsilon, ret_time=False): + # Values for an 6*8*8 grid search which was performed by Hofman et. al. + # Values obtained from Jasper Snoek + # C, alpha, epsilon + # opt_time: return time instead of performance + svm_grid_configs = [[600, 0.5, 0.01, 0.2762, 191.98], + [900, 0.8, 0.1, 0.2742, 99.976], + [2000, 5, 0.1, 0.2791, 114.41], + [5000, 1, 0.0001, 0.2577, 748.726], + [1, 5, 0.0001, 0.28714, 1048.862], + [600, 0.8, 0.1, 0.27926, 93.102], + [75, 0.5, 0.0001, 0.33702, 843.996], + [10000, 1.5, 0.01, 0.26462, 297.46], + [1000, 0.8, 0.0001, 0.2544, 1986.86666667], + [50, 0.2, 0.01, 0.39706, 226.594], + [5000, 3, 0.001, 0.26474, 814.784], + [1000000, 1, 0.01, 0.5, 156.472], + [10, 5, 0.0001, 0.28722, 1098.516], + [5, 3, 0.001, 0.34864, 305.692], + [10000, 3, 0.001, 0.26552, 873.812], + [800, 1, 0.001, 0.2785, 347.486], + [100, 0.5, 0.0001, 0.3232, 833.83], + [500, 1, 0.001, 0.2821, 381.762], + [300, 2, 0.001, 0.288, 312.296], + [600, 1.5, 0.01, 0.28258, 195.106], + [2000, 0.6, 0.01, 0.27058, 161.32], + [100, 0.5, 0.1, 0.35676, 79.6], + [400, 0.3, 0.01, 0.28054, 207.808], + [5000, 3, 0.01, 0.27486, 239.416], + [1000, 2, 0.01, 0.28104, 155.44], + [100, 0.1, 0.001, 0.34946, 482.008], + [10, 0.8, 0.001, 0.38166, 424.346], + [10, 3, 0.001, 0.31004, 475.116], + [2000, 0.4, 0.0001, 0.24785, 1623.4025], + [25, 0.2, 0.001, 0.41688, 411.466], + [3000, 0.4, 0.0001, 0.24756, 1248.566], + [600, 0.4, 0.0001, 0.27546, 857.812], + [4000, 5, 0.001, 0.26938, 831.988], + [1000, 0.9, 0.001, 0.27682, 305.498], + [7000, 5, 0.001, 0.26876, 793.69], + [600, 0.2, 0.01, 0.27266, 191.62], + [700, 0.8, 0.001, 0.27844, 330.508], + [7000, 3, 0.001, 0.26466, 848.348], + [75, 0.3, 0.0001, 0.35082, 908.324], + [1000000, 0.8, 0.1, 0.5, 90.708], + [5, 0.7, 0.001, 0.39864, 364.084], + [7000, 2, 0.001, 0.26008, 833.362], + [1, 0.6, 0.01, 0.42818, 173.37], + [7000, 0.1, 0.01, 0.25052, 209.178], + [500, 0.7, 0.01, 0.28004, 190.85], + [3000, 0.7, 0.001, 0.25062, 685.636], + [4000, 0.2, 0.0001, 0.24396, 1086.718], + [50, 0.5, 0.001, 0.35852, 425.03], + [900, 0.7, 0.001, 0.27574, 297.056], + [400, 2, 0.01, 0.28514, 178.252], + [5000, 0.1, 0.0001, 0.24196, 939.238], + [200, 0.6, 0.0001, 0.29602, 824.136], + [50, 0.5, 0.0001, 0.356, 1033.77], + [3000, 0.9, 0.01, 0.27026, 170.61], + [300, 3, 0.01, 0.28716, 178.034], + [0.1, 5, 0.01, 0.30022, 168.136], + [7000, 0.4, 0.001, 0.2464, 597.92], + [1000000, 0.8, 0.0001, 0.5, 945.362], + [200, 0.1, 0.0001, 0.29966, 837.446], + [6000, 1, 0.0001, 0.26336, 480.288], + [75, 1, 0.01, 0.33582, 186.376], + [10000, 0.4, 0.0001, 0.2461, 1121.432], + [600, 0.9, 0.001, 0.28038, 377.958], + [700, 0.8, 0.1, 0.27686, 89.688], + [100, 0.4, 0.0001, 0.32776, 854.29], + [6000, 3, 0.1, 0.27414, 159.082], + [5000, 0.6, 0.01, 0.26246, 207.476], + [3000, 0.1, 0.0001, 0.24226, 1101.012], + [7000, 0.7, 0.01, 0.24986, 359.704], + [100, 0.9, 0.1, 0.31914, 79.928], + [400, 0.5, 0.001, 0.28194, 398.602], + [1000, 5, 0.001, 0.26784, 785.06], + [1, 1.5, 0.0001, 0.34782, 1062.154], + [800, 0.7, 0.001, 0.27648, 309.812], + [4000, 0.2, 0.001, 0.24368, 645.686], + [7000, 0.6, 0.1, 0.26214, 150.098], + [1000000, 0.9, 0.001, 0.5, 280.154], + [5, 5, 0.1, 0.33336, 66.22], + [1, 1.5, 0.1, 0.39298, 68.592], + [4000, 1, 0.1, 0.27002, 95.75], + [2000, 0.9, 0.1, 0.27118, 101.142], + [900, 3, 0.1, 0.28126, 97.174], + [1000, 2, 0.0001, 0.260866666667, 1403.96], + [10000, 1.5, 0.0001, 0.2825, 1146.192], + [5000, 0.2, 0.1, 0.25934, 115.906], + [4000, 0.6, 0.001, 0.2495, 625.496], + [500, 2, 0.0001, 0.26514, 1913.09], + [2000, 0.9, 0.0001, 0.252625, 1501.0925], + [500, 0.3, 0.001, 0.27694, 401.348], + [2000, 0.2, 0.1, 0.26326, 98.796], + [1000000, 0.2, 0.1, 0.5, 114.502], + [300, 0.1, 0.001, 0.28604, 390.852], + [50, 0.4, 0.001, 0.36676, 441.684], + [100, 5, 0.0001, 0.27402, 1192.626], + [800, 2, 0.0001, 0.26402, 1703.954], + [1, 5, 0.01, 0.29758, 183.424], + [5000, 0.9, 0.001, 0.25164, 777.066], + [2000, 2, 0.001, 0.2608, 711.664], + [600, 1, 0.1, 0.28378, 54.028], + [4000, 2, 0.0001, 0.26076, 1192.924], + [25, 0.7, 0.001, 0.36862, 389.97], + [500, 0.3, 0.1, 0.27662, 86.12], + [4000, 0.4, 0.01, 0.26418, 163.332], + [500, 0.9, 0.1, 0.28126, 84.918], + [900, 0.5, 0.01, 0.27312, 187.244], + [10000, 1.5, 0.1, 0.27406, 148.158], + [900, 1.5, 0.01, 0.28038, 159.878], + [600, 0.3, 0.0001, 0.2742, 843.632], + [600, 0.1, 0.001, 0.27114, 399.358], + [200, 0.9, 0.01, 0.29442, 187.706], + [50, 0.7, 0.01, 0.35294, 198.696], + [25, 0.2, 0.1, 0.46938, 72.34], + [300, 1.5, 0.0001, 0.28738, 821.884], + [75, 0.7, 0.001, 0.32858, 419.148], + [800, 0.4, 0.01, 0.27268, 188.818], + [7000, 0.8, 0.01, 0.25148, 369.136], + [6000, 1.5, 0.1, 0.26684, 140.058], + [3000, 0.9, 0.1, 0.26778, 113.164], + [3000, 3, 0.001, 0.26436, 781.674], + [0.1, 1, 0.01, 0.38922, 150.924], + [100, 0.1, 0.01, 0.35448, 223.032], + [1000000, 0.5, 0.01, 0.5, 180.966], + [1, 0.4, 0.0001, 0.44414, 1033.23], + [600, 0.3, 0.1, 0.27302, 86.148], + [10, 5, 0.1, 0.3787, 65.432], + [3000, 5, 0.0001, 0.2695, 1539.228], + [4000, 3, 0.01, 0.2769, 191.418], + [100, 1.5, 0.1, 0.30546, 84.802], + [6000, 0.2, 0.01, 0.25666, 213.316], + [0.1, 0.3, 0.1, 0.47566, 62.724], + [200, 0.2, 0.001, 0.2981, 379.944], + [800, 0.6, 0.0001, 0.27498, 697.262], + [1, 0.1, 0.001, 0.46926, 342.85], + [4000, 0.6, 0.1, 0.26052, 116.586], + [800, 1, 0.0001, 0.27768, 672.318], + [4000, 0.5, 0.001, 0.24794, 633.722], + [3000, 0.2, 0.001, 0.25498, 463.4], + [10000, 0.8, 0.1, 0.26354, 141.786], + [1000, 0.5, 0.1, 0.2715, 96.61], + [500, 0.9, 0.001, 0.2818, 388.982], + [900, 0.5, 0.0001, 0.272566666667, 636.433333333], + [1000000, 0.7, 0.01, 0.5, 171.13], + [5000, 0.4, 0.001, 0.24662, 696.42], + [300, 0.1, 0.01, 0.28604, 188.872], + [1000, 0.4, 0.01, 0.27098, 179.646], + [25, 0.7, 0.0001, 0.36802, 831.36], + [1000000, 2, 0.1, 0.47274, 81.712], + [1000000, 0.3, 0.1, 0.5, 99.086], + [800, 0.1, 0.001, 0.26876, 303.86], + [25, 1.5, 0.001, 0.39774, 275.812], + [6000, 0.5, 0.0001, 0.24768, 1200.336], + [1000000, 2, 0.01, 0.40806, 138.358], + [0.1, 0.1, 0.01, 0.46946, 168.158], + [500, 2, 0.01, 0.28406, 180.38], + [600, 1, 0.001, 0.28062, 367.558], + [400, 0.4, 0.0001, 0.28092, 924.564], + [50, 0.4, 0.1, 0.44714, 64.674], + [300, 0.9, 0.1, 0.2877, 89.732], + [200, 2, 0.01, 0.29112, 174.154], + [1000000, 0.9, 0.1, 0.5, 97.332], + [600, 0.8, 0.01, 0.27912, 184.422], + [10, 0.2, 0.1, 0.48226, 56.632], + [2000, 1.5, 0.0001, 0.25795, 1302.53], + [6000, 0.8, 0.001, 0.25066, 653.718], + [5000, 0.8, 0.001, 0.251, 645.086], + [1, 0.1, 0.1, 0.48462, 64.426], + [100, 1.5, 0.001, 0.30438, 383.798], + [1, 0.5, 0.01, 0.43472, 177.06], + [1000000, 3, 0.1, 0.45206, 90.21], + [500, 1.5, 0.001, 0.28366, 376.972], + [7000, 0.7, 0.1, 0.2608, 137.904], + [300, 0.8, 0.0001, 0.28664, 897.056], + [900, 5, 0.01, 0.28444, 159.834], + [500, 0.2, 0.0001, 0.2753, 914.754], + [1, 0.2, 0.001, 0.4602, 344.65], + [7000, 2, 0.01, 0.26318, 340.2], + [7000, 0.9, 0.001, 0.25146, 679.86], + [50, 0.3, 0.001, 0.37742, 429.536], + [800, 0.7, 0.0001, 0.27596, 734.622], + [1, 0.4, 0.1, 0.45824, 72.28], + [300, 0.4, 0.1, 0.28392, 90.676], + [3000, 0.3, 0.01, 0.26454, 154.638], + [7000, 1.5, 0.01, 0.25772, 409.496], + [300, 0.1, 0.0001, 0.28594, 831.236], + [200, 0.7, 0.01, 0.29622, 188.1], + [5, 0.2, 0.0001, 0.4509, 974.506], + [1000, 0.9, 0.1, 0.2737, 88.956], + [6000, 1.5, 0.001, 0.257, 686.404], + [700, 0.9, 0.1, 0.27722, 89.512], + [400, 0.5, 0.0001, 0.28146, 955.994], + [4000, 0.2, 0.1, 0.25786, 113.644], + [500, 0.8, 0.0001, 0.28066, 924.76], + [10, 0.7, 0.01, 0.3929, 176.214], + [5, 0.8, 0.01, 0.39682, 164.94], + [1000000, 1.5, 0.1, 0.47812, 82.882], + [700, 0.1, 0.1, 0.2669, 88.556], + [100, 0.8, 0.001, 0.3146, 405.786], + [5, 0.6, 0.1, 0.43958, 64.786], + [25, 3, 0.01, 0.30488, 183.214], + [400, 1.5, 0.01, 0.2845, 185.428], + [50, 1.5, 0.0001, 0.31074, 1144.406], + [10000, 5, 0.001, 0.27124, 915.448], + [900, 1.5, 0.001, 0.28028, 300.058], + [100, 1.5, 0.0001, 0.30012, 1052.118], + [10000, 2, 0.0001, 0.28056, 1245.674], + [6000, 3, 0.0001, 0.26562, 1629.002], + [500, 5, 0.001, 0.27102, 742.192], + [600, 3, 0.0001, 0.26594, 1653.042], + [4000, 0.7, 0.1, 0.2623, 118.162], + [10000, 5, 0.01, 0.27556, 254.364], + [800, 5, 0.1, 0.28192, 91.156], + [300, 5, 0.001, 0.27646, 590.67], + [400, 0.2, 0.01, 0.27976, 198.17], + [700, 0.1, 0.0001, 0.269225, 717.565], + [2000, 0.4, 0.01, 0.26748, 159.678], + [0.1, 1.5, 0.0001, 0.3572, 1120.41], + [6000, 0.2, 0.001, 0.2437, 562.952], + [10, 2, 0.0001, 0.32, 1124.784], + [50, 0.5, 0.1, 0.45302, 63.464], + [400, 3, 0.0001, 0.26644, 1752.45], + [0.1, 0.5, 0.0001, 0.4376, 1166.87], + [1000, 3, 0.01, 0.2825, 162.89], + [7000, 0.1, 0.1, 0.255, 123.884], + [50, 0.6, 0.01, 0.35986, 200.666], + [700, 0.2, 0.0001, 0.27095, 727.81], + [5, 3, 0.01, 0.31416, 161.456], + [2000, 0.8, 0.001, 0.26098, 567.132], + [75, 1.5, 0.1, 0.32812, 81.152], + [3000, 1, 0.1, 0.27202, 90.778], + [400, 1.5, 0.1, 0.28474, 84.954], + [200, 0.1, 0.001, 0.30018, 382.706], + [4000, 0.1, 0.0001, 0.24168, 997.292], + [700, 0.8, 0.0001, 0.27715, 755.7525], + [1000000, 0.6, 0.1, 0.5, 88.332], + [5, 0.1, 0.1, 0.4876, 59.532], + [10000, 1.5, 0.001, 0.25714, 728.428], + [300, 0.7, 0.001, 0.28744, 360.1], + [25, 0.4, 0.01, 0.40768, 206.86], + [1000000, 0.8, 0.01, 0.5, 137.506], + [75, 0.4, 0.0001, 0.3432, 884.06], + [10000, 0.8, 0.01, 0.2555, 319.836], + [10000, 0.7, 0.0001, 0.27342, 1126.338], + [800, 2, 0.1, 0.2803, 85.47], + [1000000, 5, 0.001, 0.40736, 313.006], + [6000, 3, 0.001, 0.26476, 801.316], + [6000, 0.5, 0.1, 0.25866, 127.438], + [75, 0.9, 0.0001, 0.3195, 796.796], + [1000000, 3, 0.01, 0.3743, 152.658], + [800, 0.2, 0.01, 0.27004, 186.514], + [6000, 0.7, 0.001, 0.24972, 639.95], + [10000, 0.5, 0.1, 0.26096, 134.292], + [700, 1.5, 0.0001, 0.264075, 1922.0675], + [50, 1, 0.01, 0.35354, 200.424], + [500, 0.6, 0.1, 0.277225, 90.7025], + [1, 0.4, 0.001, 0.44334, 375.616], + [400, 5, 0.0001, 0.26774, 1558.12], + [10, 0.2, 0.01, 0.44462, 195.0], + [300, 3, 0.0001, 0.26762, 1884.616], + [50, 1.5, 0.1, 0.38136, 68.446], + [200, 5, 0.01, 0.28722, 146.236], + [700, 2, 0.0001, 0.2646, 1563.2175], + [3000, 5, 0.1, 0.27902, 132.058], + [600, 0.9, 0.0001, 0.27958, 866.892], + [75, 0.1, 0.001, 0.37422, 442.79], + [0.1, 3, 0.0001, 0.31042, 1130.442], + [400, 0.9, 0.001, 0.28388, 401.908], + [3000, 3, 0.1, 0.27484, 123.96], + [100, 0.8, 0.0001, 0.3133, 860.986], + [100, 1, 0.01, 0.32508, 182.17], + [100, 0.6, 0.001, 0.32034, 408.704], + [10, 0.3, 0.1, 0.46998, 61.478], + [6000, 0.6, 0.01, 0.25478, 272.768], + [0.1, 1.5, 0.1, 0.39805, 64.5525], + [1000, 0.6, 0.001, 0.27398, 298.708], + [6000, 0.2, 0.1, 0.2544, 120.432], + [7000, 1.5, 0.1, 0.26866, 146.586], + [25, 0.1, 0.1, 0.47186, 67.134], + [0.1, 0.8, 0.0001, 0.40828, 1117.866], + [1000, 3, 0.0001, 0.263733333333, 1483.86666667], + [3000, 0.6, 0.0001, 0.24924, 1198.446], + [5000, 2, 0.01, 0.27108, 212.174], + [300, 0.2, 0.01, 0.28646, 194.702], + [200, 5, 0.001, 0.28724, 355.718], + [500, 0.7, 0.1, 0.27752, 89.834], + [500, 1, 0.0001, 0.2824, 758.5], + [800, 1.5, 0.001, 0.281, 311.938], + [7000, 0.8, 0.1, 0.26344, 147.49], + [6000, 0.2, 0.0001, 0.2437, 916.334], + [5000, 0.7, 0.001, 0.24986, 623.874], + [25, 0.2, 0.0001, 0.41636, 913.428], + [600, 3, 0.01, 0.28424, 188.116], + [25, 0.4, 0.1, 0.4509, 67.05], + [900, 3, 0.0001, 0.2641, 1598.26333333], + [600, 0.2, 0.0001, 0.2729, 862.466], + [5, 1.5, 0.001, 0.34766, 358.646], + [0.1, 0.2, 0.01, 0.46206, 174.574], + [400, 0.2, 0.001, 0.2799, 396.37], + [400, 3, 0.1, 0.28594, 87.66], + [1000, 0.5, 0.001, 0.27266, 296.31], + [2000, 1.5, 0.01, 0.27668, 165.654], + [1000, 0.6, 0.1, 0.2718, 93.558], + [0.1, 1, 0.1, 0.44262, 48.88], + [7000, 0.4, 0.0001, 0.24664, 1141.41], + [7000, 3, 0.1, 0.27498, 157.584], + [200, 0.8, 0.01, 0.29548, 187.16], + [50, 5, 0.01, 0.29194, 176.454], + [3000, 0.8, 0.0001, 0.25168, 1197.344], + [6000, 0.1, 0.001, 0.2411, 534.628], + [75, 0.6, 0.0001, 0.3322, 859.47], + [75, 0.5, 0.1, 0.38742, 77.122], + [2000, 0.5, 0.0001, 0.249, 1647.165], + [100, 1.5, 0.01, 0.30422, 188.836], + [3000, 1, 0.01, 0.27126, 143.914], + [4000, 0.1, 0.01, 0.25918, 167.734], + [900, 0.6, 0.0001, 0.2736, 716.086666667], + [5, 0.9, 0.01, 0.38836, 164.888], + [10, 0.5, 0.001, 0.40544, 412.39], + [75, 5, 0.0001, 0.276, 1322.574], + [2000, 0.7, 0.001, 0.2669, 443.006], + [1000, 5, 0.0001, 0.2687, 1425.24], + [6000, 0.3, 0.01, 0.25478, 238.268], + [25, 1.5, 0.1, 0.37726, 72.872], + [50, 0.3, 0.01, 0.38228, 240.814], + [1000, 2, 0.001, 0.2768, 372.22], + [25, 0.4, 0.0001, 0.39388, 857.742], + [75, 0.9, 0.1, 0.34776, 76.166], + [25, 1, 0.1, 0.41652, 52.644], + [6000, 0.8, 0.1, 0.26438, 130.272], + [5000, 0.5, 0.01, 0.2592, 233.634], + [10, 0.6, 0.0001, 0.40212, 1154.298], + [3000, 1.5, 0.001, 0.25758, 723.972], + [10, 0.1, 0.1, 0.4944, 53.742], + [1000, 1.5, 0.1, 0.27826, 92.22], + [900, 0.5, 0.001, 0.27328, 303.928], + [500, 0.9, 0.01, 0.28106, 190.942], + [1000, 0.5, 0.01, 0.27242, 185.06], + [1000, 1, 0.1, 0.27934, 78.268], + [5, 3, 0.1, 0.35928, 61.924], + [400, 0.3, 0.001, 0.28038, 400.058], + [700, 5, 0.1, 0.28282, 93.462], + [50, 0.1, 0.01, 0.4098, 221.192], + [7000, 0.2, 0.0001, 0.24358, 871.402], + [800, 3, 0.0001, 0.26532, 1699.988], + [7000, 3, 0.0001, 0.26776, 1689.674], + [700, 0.4, 0.1, 0.2728, 91.716], + [600, 0.4, 0.01, 0.27518, 201.72], + [5000, 2, 0.001, 0.26014, 717.804], + [1, 1, 0.0001, 0.38674, 961.776], + [10, 0.4, 0.1, 0.47818, 57.24], + [400, 0.1, 0.01, 0.27854, 192.874], + [200, 0.4, 0.01, 0.29728, 189.836], + [300, 0.8, 0.1, 0.28672, 87.502], + [2000, 0.4, 0.001, 0.2674, 275.852], + [0.1, 0.1, 0.0001, 0.4733, 1143.676], + [10, 0.2, 0.001, 0.43718, 428.824], + [2000, 0.3, 0.1, 0.2654, 98.448], + [600, 0.1, 0.01, 0.27172, 185.26], + [600, 0.3, 0.001, 0.27446, 394.724], + [2000, 1, 0.0001, 0.26225, 1308.055], + [300, 0.3, 0.1, 0.285675, 84.735], + [5, 0.2, 0.1, 0.4721, 66.178], + [7000, 0.2, 0.01, 0.2445, 313.41], + [4000, 0.1, 0.001, 0.24178, 596.902], + [1, 0.1, 0.0001, 0.47318, 1071.218], + [700, 0.4, 0.001, 0.2741, 347.246], + [700, 0.1, 0.01, 0.26968, 188.44], + [5, 0.6, 0.0001, 0.41224, 1064.182], + [400, 1, 0.01, 0.2844, 182.836], + [400, 0.1, 0.0001, 0.27872, 925.852], + [1, 0.6, 0.0001, 0.42206, 1050.308], + [0.1, 2, 0.01, 0.3443, 150.32], + [10000, 1, 0.0001, 0.26884, 237.932], + [800, 1, 0.1, 0.2881, 91.686], + [400, 1.5, 0.0001, 0.28484, 922.604], + [10, 0.9, 0.001, 0.37518, 439.848], + [10, 1.5, 0.1, 0.41416, 62.872], + [75, 0.8, 0.1, 0.36858, 73.028], + [700, 0.4, 0.01, 0.27374, 208.716], + [5000, 2, 0.0001, 0.26066, 1253.388], + [10, 0.6, 0.001, 0.3968, 424.838], + [1000, 0.9, 0.01, 0.2773, 179.5875], + [300, 2, 0.0001, 0.27404, 1621.78], + [6000, 1, 0.01, 0.2679, 188.96], + [4000, 0.3, 0.1, 0.25802, 116.096], + [2000, 0.8, 0.01, 0.27248, 156.818], + [700, 0.3, 0.0001, 0.272225, 802.165], + [0.1, 0.2, 0.1, 0.48458, 65.068], + [800, 0.8, 0.0001, 0.27668, 814.708], + [700, 0.3, 0.001, 0.27286, 328.432], + [4000, 0.8, 0.1, 0.26138, 119.046], + [800, 5, 0.01, 0.2838, 159.182], + [1000, 1.5, 0.001, 0.27984, 299.73], + [0.1, 2, 0.0001, 0.33334, 1079.002], + [500, 0.3, 0.0001, 0.27654, 935.272], + [4000, 0.4, 0.001, 0.24668, 628.582], + [7000, 0.3, 0.0001, 0.24534, 931.676], + [300, 0.4, 0.0001, 0.2868, 845.254], + [7000, 0.5, 0.001, 0.24786, 665.792], + [200, 0.5, 0.1, 0.296525, 83.8825], + [10000, 0.9, 0.001, 0.2514, 751.582], + [200, 0.2, 0.0001, 0.29764, 846.364], + [75, 0.5, 0.001, 0.33816, 419.206], + [10000, 1, 0.01, 0.26866, 164.224], + [7000, 2, 0.0001, 0.26488, 1284.018], + [75, 0.8, 0.0001, 0.32326, 816.718], + [25, 0.6, 0.1, 0.43602, 66.078], + [50, 0.9, 0.1, 0.39444, 73.242], + [6000, 0.1, 0.1, 0.25468, 114.67], + [10000, 0.8, 0.001, 0.25038, 660.802], + [600, 2, 0.01, 0.283, 180.698], + [300, 1.5, 0.01, 0.28718, 183.712], + [800, 3, 0.001, 0.26884, 683.236], + [25, 0.5, 0.1, 0.44598, 65.5], + [100, 0.2, 0.001, 0.34058, 463.52], + [1000, 0.8, 0.001, 0.27612, 296.306], + [75, 0.4, 0.1, 0.39524, 78.414], + [6000, 2, 0.0001, 0.26566, 1269.668], + [200, 0.9, 0.1, 0.29526, 86.62], + [3000, 0.8, 0.001, 0.25204, 679.502], + [700, 3, 0.0001, 0.2658, 1701.59], + [1, 2, 0.1, 0.36774, 74.084], + [0.1, 1.5, 0.01, 0.3562, 170.108], + [5000, 0.9, 0.1, 0.26558, 128.818], + [600, 0.9, 0.01, 0.28034, 191.452], + [800, 0.3, 0.01, 0.27124, 188.02], + [200, 5, 0.1, 0.28716, 78.842], + [400, 0.4, 0.001, 0.28116, 435.502], + [300, 1, 0.0001, 0.28928, 828.222], + [500, 3, 0.001, 0.2771, 600.59], + [2000, 3, 0.1, 0.27738, 112.184], + [800, 0.6, 0.001, 0.27528, 309.874], + [7000, 0.5, 0.0001, 0.24736, 1152.222], + [4000, 5, 0.01, 0.27848, 202.646], + [1000, 0.2, 0.01, 0.2685, 160.5], + [700, 0.2, 0.001, 0.27138, 345.794], + [800, 0.9, 0.001, 0.2781, 309.204], + [900, 3, 0.01, 0.2829, 157.484], + [1, 0.7, 0.001, 0.41394, 348.66], + [0.1, 1, 0.0001, 0.39142, 1045.278], + [700, 0.5, 0.1, 0.2731, 90.812], + [25, 0.3, 0.01, 0.4156, 209.686], + [200, 1, 0.0001, 0.29942, 773.04], + [4000, 1.5, 0.0001, 0.258, 1194.092], + [700, 0.7, 0.0001, 0.276125, 764.4775], + [3000, 0.2, 0.01, 0.26238, 161.98], + [3000, 1.5, 0.01, 0.27342, 171.822], + [800, 0.8, 0.01, 0.2767, 206.558], + [400, 2, 0.0001, 0.26658, 2024.066], + [700, 2, 0.01, 0.2825, 180.924], + [1000, 0.1, 0.0001, 0.266766666667, 549.47], + [10, 3, 0.01, 0.30906, 173.374], + [75, 0.2, 0.01, 0.3706, 198.368], + [10, 0.5, 0.0001, 0.41132, 1045.728], + [75, 1, 0.001, 0.33434, 388.386], + [600, 1.5, 0.0001, 0.26348, 2057.35], + [3000, 0.2, 0.0001, 0.24392, 1139.442], + [1, 0.8, 0.1, 0.4312, 68.044], + [3000, 0.2, 0.1, 0.26184, 99.994], + [1000000, 0.5, 0.0001, 0.5, 744.08], + [10, 0.5, 0.1, 0.48128, 55.422], + [5, 5, 0.01, 0.29864, 161.186], + [10, 1, 0.01, 0.3884, 169.096], + [75, 1.5, 0.0001, 0.30842, 747.038], + [400, 5, 0.1, 0.28514, 85.422], + [500, 0.4, 0.0001, 0.27766, 927.036], + [800, 1.5, 0.01, 0.28066, 192.67], + [75, 0.6, 0.1, 0.37946, 80.014], + [50, 2, 0.001, 0.3087, 345.298], + [5000, 0.3, 0.01, 0.26076, 158.292], + [25, 0.1, 0.01, 0.44188, 216.076], + [3000, 0.9, 0.0001, 0.25242, 1197.638], + [1000, 0.1, 0.001, 0.26702, 289.136], + [1, 0.6, 0.1, 0.45142, 67.71], + [25, 0.3, 0.001, 0.40472, 399.568], + [25, 1.5, 0.01, 0.33584, 181.14], + [50, 0.8, 0.1, 0.41962, 68.658], + [6000, 0.4, 0.0001, 0.24652, 1056.842], + [75, 1, 0.0001, 0.33392, 892.074], + [50, 0.1, 0.001, 0.4023, 442.506], + [5, 0.3, 0.1, 0.46282, 67.408], + [3000, 2, 0.0001, 0.26096, 1214.92], + [75, 0.3, 0.1, 0.4037, 76.695], + [5, 0.3, 0.001, 0.4349, 334.592], + [50, 0.7, 0.001, 0.34526, 394.276], + [300, 0.6, 0.01, 0.2872, 181.992], + [300, 5, 0.0001, 0.26854, 1504.19], + [3000, 0.5, 0.1, 0.26476, 111.6], + [600, 5, 0.0001, 0.2672, 1545.094], + [25, 5, 0.0001, 0.32426, 785.124], + [50, 0.6, 0.001, 0.3513, 415.802], + [900, 1, 0.1, 0.2869, 78.98], + [6000, 5, 0.0001, 0.26836, 1504.232], + [0.1, 0.4, 0.0001, 0.44742, 1212.744], + [1, 0.1, 0.01, 0.46678, 190.252], + [50, 5, 0.001, 0.28122, 525.002], + [600, 0.5, 0.1, 0.27402, 96.22], + [0.1, 0.6, 0.01, 0.42914, 168.89], + [10, 0.8, 0.01, 0.3865, 185.748], + [0.1, 5, 0.0001, 0.291, 1158.952], + [800, 0.1, 0.1, 0.26522, 91.154], + [300, 1.5, 0.1, 0.28748, 82.438], + [400, 3, 0.001, 0.28174, 491.63], + [400, 0.2, 0.0001, 0.27942, 926.686], + [1000, 5, 0.01, 0.28328, 162.104], + [50, 0.9, 0.0001, 0.33382, 890.334], + [1000000, 0.4, 0.0001, 0.5, 700.214], + [5, 0.1, 0.0001, 0.46014, 1152.888], + [5000, 1.5, 0.0001, 0.26604, 1212.062], + [4000, 3, 0.001, 0.26524, 813.44], + [800, 1.5, 0.1, 0.27958, 86.498], + [200, 0.5, 0.001, 0.29678, 353.592], + [5000, 0.4, 0.01, 0.26206, 162.022], + [50, 1.5, 0.01, 0.32012, 189.9], + [700, 1.5, 0.01, 0.2813, 196.356], + [6000, 5, 0.01, 0.2769, 249.764], + [200, 0.7, 0.001, 0.29606, 346.064], + [5000, 5, 0.0001, 0.26812, 1683.928], + [2000, 0.7, 0.1, 0.26908, 117.7], + [1, 3, 0.001, 0.31172, 349.082], + [3000, 0.5, 0.01, 0.26724, 160.404], + [0.1, 0.9, 0.001, 0.39486, 426.182], + [6000, 0.6, 0.001, 0.24924, 611.438], + [200, 0.7, 0.0001, 0.29554, 925.326], + [500, 2, 0.001, 0.2848, 366.786], + [6000, 2, 0.1, 0.27034, 156.278], + [1, 0.4, 0.01, 0.44532, 174.68], + [5000, 0.1, 0.01, 0.25844, 167.61], + [900, 0.9, 0.001, 0.27712, 302.426], + [10, 1, 0.0001, 0.38288, 992.33], + [2000, 2, 0.1, 0.27476, 104.136], + [500, 0.8, 0.01, 0.28112, 211.398], + [300, 0.5, 0.001, 0.28712, 367.25], + [300, 0.4, 0.01, 0.287, 194.978], + [800, 0.4, 0.1, 0.27028, 88.328], + [300, 0.3, 0.001, 0.28662, 376.584], + [50, 1.5, 0.001, 0.31738, 366.8], + [0.1, 3, 0.1, 0.35762, 62.382], + [100, 0.6, 0.1, 0.3382, 79.99], + [300, 0.2, 0.001, 0.28686, 386.25], + [500, 0.1, 0.1, 0.27172, 86.518], + [600, 0.6, 0.001, 0.2775, 397.966], + [100, 0.4, 0.01, 0.33208, 229.522], + [0.1, 0.4, 0.01, 0.4461, 171.486], + [500, 0.5, 0.001, 0.27848, 403.954], + [2000, 2, 0.0001, 0.26095, 1268.1725], + [5000, 1.5, 0.001, 0.25724, 777.294], + [300, 1.5, 0.001, 0.28822, 346.746], + [10000, 0.9, 0.01, 0.25704, 315.816], + [0.1, 0.6, 0.1, 0.45686, 58.824], + [100, 0.4, 0.001, 0.32884, 417.614], + [400, 0.5, 0.01, 0.28232, 195.978], + [7000, 0.3, 0.01, 0.2452, 324.976], + [2000, 1.5, 0.001, 0.25874, 769.75], + [75, 0.7, 0.0001, 0.32746, 820.95], + [7000, 0.4, 0.1, 0.25728, 131.03], + [3000, 1, 0.0001, 0.25684, 1031.304], + [25, 0.8, 0.1, 0.38342, 79.944], + [7000, 2, 0.1, 0.2716, 145.62], + [1000, 0.7, 0.0001, 0.259833333333, 1582.12666667], + [4000, 1, 0.01, 0.2686, 180.24], + [2000, 0.9, 0.01, 0.27362, 166.51], + [800, 1.5, 0.0001, 0.26236, 1820.576], + [100, 0.2, 0.01, 0.34562, 217.342], + [4000, 1, 0.0001, 0.25598, 859.122], + [4000, 0.2, 0.01, 0.26156, 151.65], + [300, 0.8, 0.001, 0.28742, 352.662], + [2000, 0.1, 0.01, 0.2629, 147.184], + [5, 0.6, 0.001, 0.40756, 358.234], + [25, 1.5, 0.0001, 0.35548, 969.826], + [5, 2, 0.01, 0.33614, 160.208], + [1000, 0.1, 0.1, 0.2649, 90.124], + [3000, 5, 0.01, 0.27922, 197.538], + [100, 2, 0.1, 0.30396, 68.108], + [1, 0.7, 0.0001, 0.41152, 1070.634], + [2000, 0.1, 0.001, 0.26284, 270.556], + [600, 0.2, 0.1, 0.26956, 85.388], + [100, 0.7, 0.001, 0.31746, 405.118], + [10000, 0.1, 0.001, 0.24136, 574.17], + [900, 0.3, 0.0001, 0.2702, 636.793333333], + [25, 3, 0.1, 0.335, 72.238], + [600, 5, 0.001, 0.26752, 853.444], + [75, 0.1, 0.01, 0.38056, 212.738], + [10000, 0.1, 0.01, 0.24206, 331.946], + [5000, 0.3, 0.0001, 0.2454, 1048.786], + [75, 1.5, 0.01, 0.3122, 180.414], + [600, 0.8, 0.0001, 0.27882, 854.386], + [3000, 0.5, 0.0001, 0.24836, 1191.178], + [3000, 2, 0.01, 0.27458, 167.476], + [300, 0.3, 0.01, 0.2867, 192.884], + [25, 0.1, 0.0001, 0.42854, 952.794], + [400, 0.7, 0.01, 0.28258, 190.108], + [50, 5, 0.1, 0.33886, 68.894], + [75, 5, 0.001, 0.32194, 393.94], + [100, 5, 0.01, 0.28724, 182.942], + [1000000, 0.1, 0.1, 0.5, 70.518], + [10, 0.7, 0.001, 0.3897, 410.686], + [800, 1, 0.01, 0.2761, 178.224], + [300, 0.4, 0.001, 0.287, 371.626], + [2000, 3, 0.0001, 0.2646, 1409.9425], + [200, 3, 0.1, 0.28714, 83.776], + [10, 0.3, 0.0001, 0.42976, 1064.32], + [5, 0.3, 0.01, 0.4471, 186.58], + [0.1, 0.3, 0.01, 0.45306, 172.472], + [700, 0.9, 0.01, 0.27872, 190.404], + [25, 0.4, 0.001, 0.39424, 435.566], + [5, 0.2, 0.01, 0.45266, 173.456], + [75, 0.2, 0.1, 0.42646, 80.556], + [2000, 1, 0.001, 0.27244, 205.764], + [700, 1.5, 0.1, 0.28095, 93.5225], + [75, 0.7, 0.01, 0.33256, 206.332], + [2000, 0.6, 0.1, 0.26872, 109.012], + [0.1, 0.6, 0.001, 0.4213, 373.228], + [10, 0.8, 0.1, 0.48062, 50.652], + [300, 0.2, 0.1, 0.28524, 97.464], + [4000, 0.7, 0.01, 0.2663, 162.276], + [1000, 0.6, 0.0001, 0.265433333333, 1178.76333333], + [400, 0.3, 0.0001, 0.27994, 904.98], + [5000, 1, 0.1, 0.27236, 106.98], + [5000, 5, 0.01, 0.27662, 236.194], + [1, 0.2, 0.1, 0.47676, 61.724], + [900, 0.8, 0.001, 0.27668, 338.366], + [1000000, 0.1, 0.0001, 0.5, 152.856], + [800, 0.2, 0.001, 0.27018, 304.842], + [1000, 0.7, 0.001, 0.2752, 323.488], + [6000, 2, 0.001, 0.26032, 724.04], + [1000, 0.3, 0.01, 0.26954, 168.854], + [1000000, 5, 0.0001, 0.40118, 564.886], + [100, 0.6, 0.0001, 0.31892, 842.156], + [3000, 5, 0.001, 0.26808, 998.786], + [200, 0.5, 0.01, 0.2968, 202.116], + [100, 3, 0.1, 0.29562, 81.284], + [500, 0.8, 0.001, 0.28076, 405.124], + [5, 0.8, 0.1, 0.43854, 61.566], + [6000, 0.4, 0.001, 0.2463, 584.552], + [400, 0.9, 0.0001, 0.28352, 911.014], + [1000, 3, 0.001, 0.2651, 838.328], + [700, 3, 0.01, 0.28444, 163.718], + [200, 0.5, 0.0001, 0.29676, 840.092], + [200, 0.4, 0.001, 0.29712, 363.562], + [400, 0.1, 0.1, 0.277275, 90.1575], + [5, 1, 0.0001, 0.38436, 927.208], + [4000, 0.5, 0.1, 0.2599, 117.092], + [900, 0.7, 0.01, 0.27506, 186.6], + [900, 1, 0.001, 0.2774, 338.136], + [0.1, 1, 0.001, 0.38732, 369.28], + [500, 0.5, 0.0001, 0.2784, 926.586], + [7000, 0.2, 0.001, 0.24314, 557.366], + [2000, 3, 0.01, 0.28084, 167.73], + [10, 1.5, 0.001, 0.34178, 446.116], + [1000000, 0.4, 0.001, 0.5, 284.836], + [10, 1, 0.1, 0.45074, 44.92], + [1, 2, 0.0001, 0.32726, 991.374], + [100, 0.9, 0.001, 0.31214, 398.49], + [0.1, 0.7, 0.1, 0.45412, 58.87], + [6000, 1.5, 0.01, 0.2663, 232.714], + [4000, 0.9, 0.1, 0.26394, 123.406], + [800, 0.3, 0.001, 0.27176, 314.438], + [7000, 0.4, 0.01, 0.2463, 334.802], + [4000, 0.5, 0.0001, 0.24796, 1206.42], + [800, 0.8, 0.001, 0.2773, 307.122], + [500, 0.1, 0.01, 0.2742, 193.062], + [600, 0.6, 0.0001, 0.2768, 867.236], + [700, 0.9, 0.001, 0.27934, 339.444], + [900, 5, 0.001, 0.27016, 673.782], + [25, 0.9, 0.1, 0.4167, 68.048], + [3000, 1, 0.001, 0.26962, 299.682], + [5, 0.9, 0.1, 0.40614, 65.708], + [1000, 0.8, 0.01, 0.27578, 189.112], + [500, 1.5, 0.1, 0.28482, 90.764], + [50, 0.7, 0.1, 0.43114, 61.586], + [10, 0.9, 0.0001, 0.3779, 1261.55], + [500, 0.4, 0.01, 0.2779, 196.13], + [2000, 0.6, 0.001, 0.26994, 306.254], + [75, 0.1, 0.0001, 0.37276, 933.128], + [200, 1.5, 0.1, 0.29206, 83.252], + [800, 5, 0.001, 0.26794, 754.956], + [600, 0.4, 0.001, 0.27542, 394.648], + [2000, 0.4, 0.1, 0.26664, 102.026], + [1000000, 0.2, 0.01, 0.5, 183.916], + [5, 0.7, 0.1, 0.4343, 60.78], + [7000, 1, 0.01, 0.2702, 158.588], + [5000, 0.8, 0.1, 0.2635, 126.588], + [7000, 0.8, 0.0001, 0.25264, 1118.04], + [75, 5, 0.1, 0.30028, 78.936], + [500, 0.4, 0.1, 0.27672, 90.402], + [0.1, 3, 0.01, 0.31722, 183.04], + [700, 0.6, 0.1, 0.27284, 87.81], + [5000, 0.5, 0.0001, 0.24796, 1184.634], + [5000, 0.6, 0.1, 0.26366, 126.656], + [500, 0.7, 0.001, 0.27976, 385.09], + [1000000, 1.5, 0.01, 0.49762, 124.654], + [1, 3, 0.01, 0.31604, 190.664], + [700, 0.5, 0.01, 0.27502, 193.652], + [1, 0.5, 0.001, 0.43322, 351.976], + [5, 0.7, 0.0001, 0.40232, 1035.59], + [500, 0.2, 0.001, 0.27568, 391.312], + [25, 1, 0.001, 0.36832, 361.092], + [400, 2, 0.1, 0.28216, 81.332], + [5, 3, 0.0001, 0.30488, 1159.526], + [0.1, 0.5, 0.1, 0.4743, 58.502], + [6000, 0.3, 0.001, 0.24502, 567.554], + [1, 0.5, 0.0001, 0.43388, 1087.284], + [900, 0.1, 0.001, 0.26776, 298.986], + [400, 0.6, 0.0001, 0.28224, 933.726], + [900, 0.1, 0.1, 0.265, 97.4975], + [50, 5, 0.0001, 0.27566, 1293.048], + [5000, 3, 0.0001, 0.26562, 1611.418], + [10000, 1, 0.1, 0.28032, 109.504], + [10, 0.9, 0.1, 0.46826, 61.228], + [700, 0.5, 0.001, 0.27522, 327.876], + [6000, 5, 0.001, 0.2679, 842.408], + [700, 2, 0.1, 0.2816, 84.222], + [300, 0.5, 0.0001, 0.28678, 869.392], + [6000, 0.4, 0.1, 0.25764, 123.252], + [75, 3, 0.1, 0.29808, 84.79], + [10, 0.8, 0.0001, 0.38592, 1187.246], + [100, 1, 0.0001, 0.32384, 834.042], + [400, 0.6, 0.001, 0.28266, 385.716], + [10, 0.5, 0.01, 0.41406, 179.11], + [0.1, 0.8, 0.1, 0.44602, 60.576], + [800, 2, 0.01, 0.28262, 149.586], + [10000, 0.9, 0.0001, 0.28978, 1136.934], + [5, 1, 0.01, 0.39272, 155.548], + [25, 0.7, 0.01, 0.3753, 227.884], + [7000, 0.2, 0.1, 0.25526, 127.32], + [400, 0.1, 0.001, 0.27908, 389.69], + [100, 2, 0.0001, 0.2855, 1342.104], + [200, 0.9, 0.001, 0.29516, 322.786], + [900, 5, 0.0001, 0.2681, 1377.79333333], + [200, 1, 0.01, 0.30064, 182.916], + [50, 0.8, 0.01, 0.34418, 215.614], + [300, 0.5, 0.01, 0.2875, 191.246], + [1, 0.7, 0.01, 0.41388, 178.784], + [7000, 5, 0.0001, 0.26844, 1511.754], + [900, 0.9, 0.0001, 0.255766666667, 2085.64666667], + [75, 3, 0.001, 0.32722, 428.408], + [10000, 3, 0.01, 0.26828, 313.056], + [3000, 2, 0.1, 0.27448, 114.978], + [1, 0.8, 0.01, 0.4089, 177.198], + [3000, 0.7, 0.1, 0.2679, 118.854], + [4000, 0.3, 0.01, 0.26284, 156.764], + [25, 2, 0.0001, 0.3441, 909.126], + [400, 1.5, 0.001, 0.2853, 379.38], + [1, 1.5, 0.001, 0.35434, 362.86], + [800, 0.7, 0.01, 0.27586, 187.5], + [500, 0.7, 0.0001, 0.28006, 943.238], + [6000, 0.7, 0.0001, 0.25076, 1110.904], + [500, 0.9, 0.0001, 0.28142, 941.56], + [800, 0.2, 0.0001, 0.2701, 651.41], + [10, 1, 0.001, 0.37668, 386.812], + [5000, 0.5, 0.001, 0.2482, 584.28], + [10000, 0.4, 0.1, 0.25832, 138.412], + [100, 0.3, 0.001, 0.3333, 422.244], + [10000, 0.6, 0.0001, 0.254, 1078.206], + [4000, 0.4, 0.1, 0.26012, 116.442], + [6000, 0.9, 0.01, 0.2623, 221.634], + [5, 0.9, 0.0001, 0.38618, 1200.626], + [4000, 0.6, 0.01, 0.26582, 165.468], + [5000, 1, 0.01, 0.26698, 198.29], + [0.1, 5, 0.1, 0.3441, 59.698], + [800, 0.1, 0.0001, 0.26838, 711.628], + [900, 0.3, 0.1, 0.26824, 98.986], + [5000, 0.1, 0.001, 0.24166, 558.132], + [100, 0.1, 0.1, 0.3972, 85.508], + [900, 0.4, 0.01, 0.27182, 184.53], + [700, 0.9, 0.0001, 0.277725, 810.8575], + [700, 3, 0.1, 0.28146, 88.434], + [2000, 1, 0.1, 0.2733, 85.972], + [10000, 5, 0.1, 0.28042, 167.404], + [5, 0.5, 0.001, 0.41648, 365.746], + [50, 1, 0.0001, 0.3471, 967.686], + [10000, 0.5, 0.0001, 0.2471, 1077.628], + [6000, 0.9, 0.001, 0.2517, 626.65], + [10, 2, 0.1, 0.45516, 45.912], + [300, 0.6, 0.1, 0.28728, 89.146], + [600, 0.7, 0.001, 0.27844, 369.498], + [900, 0.6, 0.1, 0.27334, 93.652], + [3000, 0.5, 0.001, 0.24868, 706.666], + [400, 0.8, 0.0001, 0.28288, 921.16], + [7000, 0.6, 0.001, 0.24936, 607.588], + [800, 0.2, 0.1, 0.26838, 86.472], + [10000, 0.8, 0.0001, 0.27032, 1147.492], + [3000, 0.3, 0.0001, 0.24584, 1286.306], + [200, 0.1, 0.01, 0.30184, 194.42], + [25, 0.2, 0.01, 0.43052, 204.764], + [5, 0.3, 0.0001, 0.44132, 1021.024], + [700, 0.6, 0.0001, 0.275225, 789.6675], + [300, 1, 0.1, 0.29412, 73.232], + [100, 3, 0.001, 0.28634, 493.786], + [75, 0.2, 0.0001, 0.3602, 862.904], + [1, 0.6, 0.001, 0.4225, 353.728], + [600, 0.1, 0.0001, 0.27116, 809.304], + [0.1, 2, 0.1, 0.39858, 52.04], + [400, 0.7, 0.001, 0.2828, 392.862], + [1000000, 2, 0.0001, 0.42602, 514.108], + [900, 0.2, 0.01, 0.26916, 172.61], + [5, 0.7, 0.01, 0.40932, 170.262], + [25, 2, 0.001, 0.39002, 269.67], + [400, 0.5, 0.1, 0.28174, 90.886], + [10, 0.2, 0.0001, 0.43962, 961.244], + [300, 0.7, 0.01, 0.28742, 185.994], + [1000, 0.2, 0.001, 0.26874, 325.168], + [4000, 0.3, 0.0001, 0.24542, 996.174], + [5000, 0.7, 0.1, 0.26352, 120.336], + [5000, 0.2, 0.001, 0.24324, 634.6], + [10, 3, 0.0001, 0.30214, 1093.328], + [7000, 1, 0.0001, 0.26272, 424.826], + [100, 0.7, 0.0001, 0.31596, 864.23], + [0.1, 0.7, 0.01, 0.41802, 168.444], + [10, 0.9, 0.01, 0.37986, 178.462], + [1000000, 1.5, 0.001, 0.48788, 387.352], + [75, 2, 0.1, 0.31056, 72.662], + [6000, 3, 0.01, 0.27282, 258.288], + [0.1, 0.7, 0.0001, 0.41688, 1152.06], + [10, 0.6, 0.1, 0.47892, 55.994], + [100, 0.1, 0.0001, 0.34888, 923.942], + [4000, 0.6, 0.0001, 0.24914, 1113.7], + [200, 0.8, 0.1, 0.2959, 84.91], + [50, 3, 0.001, 0.2926, 478.132], + [4000, 0.8, 0.001, 0.25136, 645.918], + [0.1, 0.1, 0.001, 0.46842, 401.66], + [900, 0.2, 0.0001, 0.269333333333, 628.16], + [0.1, 0.5, 0.01, 0.4332, 170.198], + [4000, 0.9, 0.001, 0.2523, 739.958], + [5000, 1.5, 0.01, 0.26838, 225.152], + [400, 2, 0.001, 0.28562, 353.47], + [75, 3, 0.01, 0.298, 180.122], + [900, 0.6, 0.001, 0.27466, 310.13], + [50, 0.3, 0.0001, 0.3751, 989.71], + [1000000, 0.8, 0.001, 0.5, 293.656], + [100, 2, 0.001, 0.29794, 387.326], + [300, 5, 0.01, 0.28578, 148.172], + [7000, 0.5, 0.01, 0.24834, 340.102], + [25, 0.1, 0.001, 0.42824, 441.102], + [800, 0.4, 0.001, 0.27274, 300.816], + [10000, 0.1, 0.1, 0.2529, 119.038], + [1000, 0.2, 0.0001, 0.268366666667, 700.76], + [75, 0.4, 0.001, 0.34458, 421.642], + [1000, 2, 0.1, 0.2788, 91.44], + [1000000, 3, 0.0001, 0.41112, 474.012], + [10, 2, 0.001, 0.32652, 418.05], + [100, 0.5, 0.01, 0.32816, 207.79], + [700, 0.6, 0.01, 0.27578, 190.46], + [200, 0.4, 0.0001, 0.29706, 844.122], + [50, 3, 0.1, 0.34192, 71.846], + [200, 3, 0.001, 0.28944, 331.724], + [1000000, 0.3, 0.01, 0.5, 185.472], + [5000, 0.6, 0.0001, 0.24886, 1109.19], + [75, 5, 0.01, 0.28944, 174.842], + [10000, 0.6, 0.001, 0.24862, 655.32], + [100, 1, 0.1, 0.32568, 69.28], + [4000, 0.3, 0.001, 0.2452, 607.508], + [1000000, 0.3, 0.001, 0.5, 286.296], + [5, 0.5, 0.01, 0.4255, 173.882], + [75, 0.9, 0.01, 0.32616, 201.38], + [1000, 1.5, 0.0001, 0.2583, 1784.29666667], + [1, 0.9, 0.1, 0.42466, 67.466], + [300, 1, 0.001, 0.2898, 367.256], + [400, 0.4, 0.01, 0.2818, 196.702], + [50, 0.2, 0.0001, 0.38622, 1070.496], + [700, 3, 0.001, 0.2686, 729.396], + [300, 0.7, 0.1, 0.28576, 86.99], + [1000000, 1.5, 0.0001, 0.49076, 617.896], + [200, 0.9, 0.0001, 0.29428, 814.972], + [50, 0.4, 0.01, 0.37258, 220.142], + [1000, 0.7, 0.1, 0.27324, 91.482], + [3000, 0.6, 0.001, 0.24968, 652.682], + [1000, 1, 0.0001, 0.276066666667, 592.426666667], + [900, 5, 0.1, 0.28214, 98.342], + [3000, 0.7, 0.0001, 0.25082, 1230.43], + [7000, 5, 0.1, 0.27998, 154.486], + [200, 0.4, 0.1, 0.2941, 91.948], + [2000, 0.6, 0.0001, 0.2498, 1601.1825], + [50, 2, 0.01, 0.31218, 172.738], + [800, 0.3, 0.0001, 0.2715, 655.716], + [50, 0.9, 0.01, 0.34062, 210.148], + [100, 0.3, 0.01, 0.338, 219.066], + [400, 0.6, 0.01, 0.28214, 191.192], + [25, 0.7, 0.1, 0.43346, 63.682], + [1, 2, 0.01, 0.34318, 157.166], + [200, 0.2, 0.1, 0.3005, 89.43], + [2000, 1.5, 0.1, 0.27326, 119.99], + [2000, 0.3, 0.01, 0.26606, 154.594], + [600, 5, 0.01, 0.28478, 165.648], + [400, 0.4, 0.1, 0.28006, 88.578], + [300, 1, 0.01, 0.2899, 165.49], + [100, 3, 0.0001, 0.27662, 1497.498], + [200, 0.8, 0.001, 0.29538, 350.604], + [100, 0.9, 0.01, 0.3136, 196.822], + [10, 3, 0.1, 0.40006, 53.434], + [900, 0.7, 0.1, 0.27322, 94.258], + [100, 0.5, 0.001, 0.32406, 395.308], + [700, 0.1, 0.001, 0.2696, 336.958], + [7000, 1.5, 0.001, 0.25746, 807.698], + [10000, 3, 0.0001, 0.27012, 1494.092], + [1000000, 0.2, 0.001, 0.5, 286.64], + [400, 0.8, 0.001, 0.28344, 394.92], + [500, 5, 0.0001, 0.26744, 1643.9], + [1000000, 3, 0.001, 0.3913, 297.292], + [700, 1.5, 0.001, 0.28164, 338.14], + [1, 0.9, 0.001, 0.3948, 342.218], + [2000, 0.7, 0.0001, 0.251025, 1605.2675], + [700, 2, 0.001, 0.28264, 350.05], + [5, 5, 0.0001, 0.28868, 1156.032], + [200, 1.5, 0.001, 0.29284, 304.968], + [600, 0.9, 0.1, 0.27882, 84.56], + [6000, 0.7, 0.01, 0.2603, 229.446], + [100, 2, 0.01, 0.29868, 172.308], + [200, 0.2, 0.01, 0.29912, 189.606], + [600, 1, 0.0001, 0.27994, 807.098], + [50, 0.6, 0.1, 0.44374, 64.918], + [800, 3, 0.01, 0.28372, 162.956], + [75, 0.7, 0.1, 0.36432, 74.894], + [1000000, 0.2, 0.0001, 0.5, 630.72], + [10, 0.7, 0.0001, 0.3947, 1307.996], + [6000, 1, 0.1, 0.2681, 109.674], + [1, 3, 0.1, 0.34516, 75.282], + [3000, 0.1, 0.01, 0.2607, 161.56], + [500, 0.6, 0.001, 0.2793, 398.692], + [1, 1.5, 0.01, 0.36092, 178.768], + [800, 0.7, 0.1, 0.27452, 87.044], + [2000, 0.5, 0.1, 0.26826, 103.134], + [50, 0.2, 0.1, 0.46536, 63.04], + [6000, 0.4, 0.01, 0.25522, 237.966], + [700, 1, 0.01, 0.27778, 194.92], + [25, 0.8, 0.001, 0.36226, 358.518], + [300, 0.5, 0.1, 0.28722, 82.508], + [800, 0.5, 0.001, 0.27382, 306.05], + [5000, 0.9, 0.0001, 0.25806, 1134.084], + [1, 5, 0.001, 0.29414, 370.214], + [4000, 3, 0.1, 0.27302, 138.12], + [3000, 0.3, 0.1, 0.26374, 101.466], + [900, 0.4, 0.0001, 0.2715, 649.23], + [10, 0.7, 0.1, 0.47822, 51.954], + [1, 0.2, 0.0001, 0.46368, 1054.878], + [100, 0.7, 0.1, 0.33174, 81.142], + [1000000, 0.3, 0.0001, 0.5, 592.446], + [600, 0.4, 0.1, 0.2745, 85.688], + [50, 0.8, 0.001, 0.33976, 401.61], + [1000, 0.2, 0.1, 0.2659, 90.218], + [7000, 0.3, 0.001, 0.24494, 598.638], + [25, 0.6, 0.0001, 0.37596, 832.832], + [400, 0.6, 0.1, 0.28198, 89.902], + [2000, 2, 0.01, 0.27766, 153.04], + [200, 5, 0.0001, 0.27176, 1806.102], + [1000, 1, 0.001, 0.27696, 322.304], + [600, 0.7, 0.01, 0.2781, 190.826], + [75, 0.1, 0.1, 0.42998, 75.72], + [200, 0.6, 0.001, 0.29642, 352.51], + [5, 2, 0.001, 0.3311, 341.924], + [0.1, 0.7, 0.001, 0.4134, 397.274], + [3000, 0.6, 0.1, 0.2681, 115.338], + [6000, 0.5, 0.001, 0.24824, 582.726], + [0.1, 0.4, 0.1, 0.4713, 68.072], + [7000, 0.5, 0.1, 0.2587, 132.888], + [900, 1, 0.01, 0.276, 172.26], + [300, 0.7, 0.0001, 0.28646, 899.854], + [4000, 1.5, 0.01, 0.27096, 178.934], + [2000, 0.2, 0.001, 0.26458, 271.194], + [10, 0.6, 0.01, 0.4043, 175.698], + [500, 0.2, 0.01, 0.27588, 195.34], + [10, 0.3, 0.001, 0.42636, 451.284], + [600, 1.5, 0.1, 0.28152, 92.638], + [5, 0.4, 0.0001, 0.43032, 959.25], + [2000, 0.3, 0.001, 0.26586, 281.396], + [50, 2, 0.1, 0.36454, 59.206], + [3000, 0.4, 0.01, 0.26562, 164.406], + [400, 5, 0.01, 0.28514, 185.678], + [25, 2, 0.01, 0.32052, 168.026], + [75, 0.3, 0.001, 0.35208, 428.454], + [400, 0.9, 0.01, 0.283675, 204.375], + [700, 0.5, 0.0001, 0.274175, 762.55], + [5000, 0.1, 0.1, 0.25874, 114.512], + [50, 0.7, 0.0001, 0.34302, 917.336], + [400, 1, 0.0001, 0.28498, 797.834], + [400, 0.7, 0.0001, 0.28294, 924.09], + [5000, 5, 0.001, 0.2692, 821.262], + [25, 1, 0.01, 0.37202, 198.388], + [600, 5, 0.1, 0.28294, 91.746], + [800, 0.4, 0.0001, 0.27238, 697.346], + [2000, 0.1, 0.1, 0.2618, 95.55], + [1000000, 0.5, 0.001, 0.5, 285.638], + [500, 1.5, 0.01, 0.28332, 197.076], + [10000, 2, 0.01, 0.26388, 339.25], + [200, 3, 0.0001, 0.27642, 1491.856], + [5000, 0.4, 0.0001, 0.24666, 1026.348], + [500, 0.8, 0.1, 0.2789, 98.748], + [300, 0.6, 0.0001, 0.2867, 870.246], + [1, 0.3, 0.01, 0.44994, 180.756], + [10000, 5, 0.0001, 0.27162, 1332.466], + [7000, 1, 0.001, 0.26652, 266.122], + [700, 5, 0.001, 0.27086, 743.828], + [2000, 0.9, 0.001, 0.25782, 626.026], + [4000, 1, 0.001, 0.26218, 386.334], + [200, 2, 0.1, 0.28926, 76.676], + [6000, 5, 0.1, 0.27894, 167.252], + [500, 0.5, 0.01, 0.27828, 193.274], + [1, 2, 0.001, 0.33426, 326.492], + [25, 5, 0.001, 0.37526, 267.138], + [5000, 0.7, 0.0001, 0.25188, 1149.17], + [500, 0.1, 0.0001, 0.27402, 903.586], + [5, 0.4, 0.01, 0.43206, 181.42], + [700, 0.2, 0.01, 0.27104, 184.49], + [7000, 0.7, 0.0001, 0.26038, 1184.366], + [1000000, 0.7, 0.0001, 0.5, 937.806], + [10000, 0.7, 0.1, 0.26502, 138.042], + [50, 0.8, 0.0001, 0.33768, 920.122], + [1000000, 0.4, 0.1, 0.5, 94.278], + [200, 0.1, 0.1, 0.2998, 84.12], + [600, 1, 0.01, 0.27946, 191.316], + [300, 3, 0.1, 0.28736, 85.87], + [800, 0.6, 0.1, 0.27386, 91.616], + [75, 0.4, 0.01, 0.34828, 205.254], + [2000, 0.2, 0.0001, 0.24555, 1501.525], + [4000, 1.5, 0.001, 0.2577, 739.756], + [10000, 0.1, 0.0001, 0.24178, 884.736], + [25, 0.8, 0.01, 0.37338, 200.534], + [2000, 0.5, 0.001, 0.26852, 280.408], + [700, 0.4, 0.0001, 0.273425, 749.8925], + [5, 0.9, 0.001, 0.38372, 346.818], + [5000, 0.6, 0.001, 0.24936, 672.644], + [1000000, 1, 0.001, 0.5, 241.932], + [5000, 0.5, 0.1, 0.26154, 119.208], + [800, 0.3, 0.1, 0.27096, 95.728], + [50, 0.6, 0.0001, 0.3492, 972.898], + [75, 0.8, 0.01, 0.32876, 189.114], + [900, 1.5, 0.0001, 0.2597, 1681.10333333], + [10000, 0.3, 0.001, 0.24512, 549.748], + [500, 0.2, 0.1, 0.27362, 89.64], + [5, 0.6, 0.01, 0.4147, 187.312], + [6000, 0.8, 0.01, 0.25792, 256.226], + [50, 0.5, 0.01, 0.3633, 225.63], + [25, 1, 0.0001, 0.36562, 766.336], + [50, 1, 0.1, 0.42188, 61.778], + [4000, 5, 0.0001, 0.26902, 1287.17], + [4000, 2, 0.1, 0.26876, 126.878], + [10000, 0.7, 0.001, 0.24966, 648.432], + [1000000, 5, 0.01, 0.40782, 164.792], + [75, 0.6, 0.01, 0.33574, 201.58], + [200, 0.8, 0.0001, 0.2949, 811.482], + [3000, 0.8, 0.1, 0.2685, 113.084], + [500, 5, 0.01, 0.2854, 184.386], + [10, 5, 0.001, 0.29568, 448.936], + [900, 0.7, 0.0001, 0.274433333333, 752.956666667], + [3000, 3, 0.0001, 0.26484, 1455.642], + [800, 0.6, 0.01, 0.27466, 185.232], + [3000, 2, 0.001, 0.26052, 737.314], + [900, 0.3, 0.001, 0.27084, 308.846], + [600, 3, 0.1, 0.28134, 86.322], + [600, 2, 0.001, 0.28334, 370.98], + [5000, 0.3, 0.001, 0.24512, 654.348], + [10000, 0.3, 0.0001, 0.2453, 982.958], + [7000, 0.6, 0.0001, 0.24884, 1098.944], + [25, 0.5, 0.0001, 0.3842, 864.636], + [0.1, 0.1, 0.1, 0.49314, 57.258], + [400, 5, 0.001, 0.27158, 739.132], + [10000, 0.3, 0.1, 0.25726, 126.818], + [600, 0.6, 0.01, 0.27706, 185.206], + [4000, 0.9, 0.01, 0.26692, 186.9], + [600, 0.7, 0.0001, 0.27768, 863.284], + [200, 0.3, 0.001, 0.29762, 366.65], + [5, 1, 0.001, 0.3813, 319.426], + [1000000, 0.9, 0.0001, 0.5, 928.034], + [900, 0.8, 0.0001, 0.261333333333, 1660.26666667], + [1000, 5, 0.1, 0.28112, 97.866], + [5000, 0.8, 0.01, 0.26328, 208.986], + [3000, 0.1, 0.001, 0.25294, 437.07], + [75, 2, 0.001, 0.34188, 293.594], + [10000, 0.6, 0.1, 0.26244, 133.818], + [2000, 0.3, 0.0001, 0.247325, 1409.0175], + [75, 0.3, 0.01, 0.3593, 202.536], + [600, 0.1, 0.1, 0.26942, 87.906], + [600, 0.3, 0.01, 0.2745, 191.162], + [900, 2, 0.0001, 0.2615, 1549.42], + [10, 0.4, 0.001, 0.41518, 422.25], + [900, 2, 0.01, 0.28182, 159.49], + [1, 0.3, 0.1, 0.47044, 64.492], + [2000, 0.7, 0.01, 0.27174, 165.078], + [0.1, 0.6, 0.0001, 0.42714, 1138.712], + [10, 0.4, 0.0001, 0.41956, 995.474], + [900, 2, 0.001, 0.27722, 375.43], + [600, 2, 0.1, 0.28232, 84.838], + [100, 3, 0.01, 0.29276, 187.61], + [5000, 2, 0.1, 0.27428, 152.904], + [5, 1.5, 0.01, 0.35362, 169.582], + [100, 5, 0.1, 0.28992, 77.758], + [1000000, 0.9, 0.01, 0.5, 148.324], + [1000, 1, 0.01, 0.27556, 172.896], + [10000, 0.4, 0.001, 0.24626, 628.786], + [1, 0.5, 0.1, 0.4674, 63.166], + [10000, 1, 0.001, 0.26832, 185.464], + [800, 0.9, 0.01, 0.27794, 190.624], + [5, 0.5, 0.1, 0.4578, 63.876], + [6000, 0.9, 0.0001, 0.26106, 1177.182], + [10, 2, 0.01, 0.33096, 156.662], + [1000, 0.8, 0.1, 0.2737, 94.394], + [1, 0.8, 0.001, 0.40362, 366.294], + [5000, 0.2, 0.01, 0.26004, 159.232], + [1000000, 0.5, 0.1, 0.5, 100.744], + [3000, 0.8, 0.01, 0.27014, 171.012], + [5, 5, 0.001, 0.33466, 311.576], + [2000, 1, 0.01, 0.27308, 141.104], + [300, 0.3, 0.0001, 0.28654, 890.966], + [1000, 1.5, 0.01, 0.27984, 158.248], + [100, 5, 0.001, 0.27954, 500.72], + [7000, 1.5, 0.0001, 0.26804, 1142.934], + [600, 0.2, 0.001, 0.27292, 398.208], + [6000, 0.3, 0.0001, 0.24546, 1000.238], + [1000, 0.4, 0.1, 0.2715, 91.888], + [10000, 0.3, 0.01, 0.24476, 355.386], + [3000, 0.6, 0.01, 0.26838, 157.41], + [700, 0.7, 0.001, 0.27704, 358.83], + [500, 3, 0.1, 0.28218, 84.64], + [7000, 0.1, 0.0001, 0.24174, 883.884], + [200, 1.5, 0.0001, 0.2924, 759.53], + [7000, 0.7, 0.001, 0.24988, 690.188], + [800, 3, 0.1, 0.2811, 95.036], + [5000, 3, 0.1, 0.2751, 143.092], + [600, 0.8, 0.001, 0.27946, 373.264], + [5000, 5, 0.1, 0.27954, 156.902], + [25, 5, 0.01, 0.29264, 182.294], + [500, 0.5, 0.1, 0.27756, 84.498], + [800, 0.9, 0.1, 0.27502, 88.914], + [7000, 0.8, 0.001, 0.25026, 645.568], + [500, 1, 0.1, 0.28366, 59.952], + [7000, 0.3, 0.1, 0.2568, 122.306], + [6000, 1, 0.001, 0.26664, 267.806], + [1000, 0.3, 0.001, 0.26994, 293.288], + [2000, 0.5, 0.01, 0.269675, 161.0725], + [1, 0.8, 0.0001, 0.40154, 1007.07], + [700, 0.7, 0.01, 0.27678, 193.244], + [1000000, 1, 0.1, 0.5, 98.16], + [1000000, 1, 0.0001, 0.5, 551.464], + [300, 2, 0.01, 0.28792, 175.738], + [3000, 3, 0.01, 0.27734, 178.99], + [75, 2, 0.01, 0.30636, 163.706], + [4000, 2, 0.001, 0.2602, 752.348], + [900, 0.8, 0.01, 0.27606, 191.036], + [1000000, 2, 0.001, 0.43144, 288.618], + [2000, 3, 0.001, 0.26476, 809.162], + [50, 0.9, 0.001, 0.33526, 386.42], + [6000, 0.6, 0.1, 0.25934, 126.272], + [1000, 0.9, 0.0001, 0.254866666667, 1979.02666667], + [0.1, 3, 0.001, 0.31316, 463.938], + [500, 0.6, 0.0001, 0.27926, 930.308], + [7000, 0.6, 0.01, 0.24958, 345.62], + [6000, 2, 0.01, 0.26492, 294.578], + [75, 0.8, 0.001, 0.32452, 407.924], + [5, 2, 0.0001, 0.32516, 1014.256], + [1, 0.3, 0.0001, 0.45462, 1099.228], + [6000, 0.3, 0.1, 0.25578, 123.272], + [6000, 0.6, 0.0001, 0.25206, 1099.786], + [200, 2, 0.001, 0.29122, 310.672], + [300, 0.9, 0.01, 0.28798, 186.704], + [1000, 0.4, 0.0001, 0.271066666667, 619.126666667], + [7000, 0.9, 0.01, 0.25654, 312.386], + [100, 0.6, 0.01, 0.32282, 205.416], + [10000, 0.6, 0.01, 0.24862, 416.906], + [500, 0.4, 0.001, 0.27782, 438.408], + [1000000, 0.7, 0.1, 0.5, 104.332], + [900, 0.9, 0.01, 0.27708, 184.072], + [2000, 0.8, 0.0001, 0.251625, 1521.055], + [300, 0.2, 0.0001, 0.28658, 858.066], + [50, 0.3, 0.1, 0.46048, 61.21], + [0.1, 0.9, 0.01, 0.40586, 174.718], + [5, 0.4, 0.1, 0.46744, 62.868], + [2000, 5, 0.0001, 0.2704, 1425.32], + [1, 1, 0.1, 0.42018, 55.71], + [0.1, 2, 0.001, 0.33648, 341.982], + [10000, 0.2, 0.0001, 0.24312, 888.092], + [300, 0.9, 0.001, 0.28762, 344.586], + [200, 2, 0.0001, 0.285, 1334.518], + [700, 0.7, 0.1, 0.27408, 88.412], + [5000, 0.7, 0.01, 0.2633, 214.44], + [1, 1, 0.001, 0.38912, 300.174], + [6000, 0.5, 0.01, 0.25344, 281.848], + [3000, 0.9, 0.001, 0.25284, 712.316], + [5, 0.8, 0.0001, 0.39384, 1228.768], + [800, 0.9, 0.0001, 0.2689, 1342.402], + [75, 0.2, 0.001, 0.36206, 420.58], + [7000, 3, 0.01, 0.268, 326.912], + [500, 5, 0.1, 0.28222, 83.39], + [900, 1.5, 0.1, 0.27794, 91.02], + [700, 5, 0.0001, 0.267875, 1574.455], + [50, 3, 0.01, 0.3007, 168.09], + [600, 2, 0.0001, 0.26486, 1867.686], + [3000, 0.3, 0.001, 0.24994, 682.044], + [700, 0.3, 0.01, 0.2729, 196.1], + [1, 3, 0.0001, 0.3045, 1023.176], + [2000, 0.2, 0.01, 0.26422, 147.94], + [4000, 1.5, 0.1, 0.26866, 127.972], + [1, 0.3, 0.001, 0.4523, 360.758], + [1000000, 0.4, 0.01, 0.5, 182.784], + [100, 0.9, 0.0001, 0.31106, 856.688], + [7000, 0.9, 0.1, 0.265, 134.972], + [25, 5, 0.1, 0.33184, 74.256], + [5000, 1.5, 0.1, 0.268, 138.442], + [4000, 0.9, 0.0001, 0.25332, 1158.6], + [75, 0.6, 0.001, 0.33322, 418.482], + [900, 0.1, 0.01, 0.26734, 165.618], + [900, 0.1, 0.0001, 0.267533333333, 607.793333333], + [0.1, 0.3, 0.0001, 0.4572, 1165.654], + [400, 0.7, 0.1, 0.28154, 88.258], + [200, 0.3, 0.0001, 0.29718, 802.894], + [900, 0.2, 0.1, 0.26608, 88.58], + [5, 0.1, 0.001, 0.4545, 350.72], + [5000, 0.8, 0.0001, 0.25116, 1185.046], + [10000, 2, 0.1, 0.27674, 158.698], + [6000, 0.9, 0.1, 0.26248, 127.374], + [400, 1, 0.1, 0.28668, 70.84], + [2000, 5, 0.001, 0.26822, 842.558], + [400, 1, 0.001, 0.2852, 392.806], + [600, 0.5, 0.0001, 0.27612, 850.596], + [900, 1, 0.0001, 0.2764, 595.7], + [700, 1, 0.1, 0.29358, 79.154], + [700, 0.6, 0.001, 0.27606, 338.544], + [400, 0.9, 0.1, 0.284675, 88.84], + [25, 0.5, 0.001, 0.38518, 390.786], + [25, 0.3, 0.0001, 0.4047, 861.23], + [6000, 0.7, 0.1, 0.26084, 127.19], + [1000, 0.7, 0.01, 0.27508, 182.732], + [800, 0.5, 0.01, 0.27376, 193.84], + [4000, 0.1, 0.1, 0.25742, 107.622], + [5000, 0.4, 0.1, 0.26054, 121.938], + [50, 0.2, 0.001, 0.3883, 486.984], + [50, 1, 0.001, 0.34926, 386.432], + [200, 3, 0.01, 0.2898, 174.974], + [200, 0.6, 0.01, 0.29656, 193.826], + [50, 0.1, 0.1, 0.47024, 63.298], + [5, 1, 0.1, 0.43308, 62.966], + [4000, 2, 0.01, 0.2728, 180.1], + [10000, 0.2, 0.001, 0.2433, 526.762], + [200, 1.5, 0.01, 0.29302, 180.636], + [700, 0.8, 0.01, 0.27808, 194.188], + [1, 0.9, 0.01, 0.39834, 181.634], + [1000000, 0.6, 0.01, 0.5, 159.072], + [100, 0.8, 0.01, 0.31706, 207.868], + [5, 0.4, 0.001, 0.4254, 356.88], + [900, 0.9, 0.1, 0.27328, 88.198], + [900, 0.3, 0.01, 0.27014, 177.064], + [900, 0.4, 0.001, 0.27168, 302.282], + [200, 0.7, 0.1, 0.29492, 89.816], + [7000, 1, 0.1, 0.26866, 121.104], + [3000, 1.5, 0.0001, 0.25838, 1167.57], + [1000, 0.1, 0.01, 0.26648, 150.268], + [900, 0.4, 0.1, 0.27052, 88.124], + [800, 0.8, 0.1, 0.27528, 89.906], + [300, 5, 0.1, 0.28652, 81.852], + [1000, 3, 0.1, 0.281, 93.516], + [0.1, 0.5, 0.001, 0.43142, 395.082], + [25, 0.5, 0.01, 0.39414, 213.126], + [400, 3, 0.01, 0.28582, 189.45], + [0.1, 0.3, 0.001, 0.45076, 415.572], + [7000, 0.9, 0.0001, 0.2568, 1110.788], + [200, 0.3, 0.01, 0.29788, 196.306], + [300, 0.9, 0.0001, 0.28726, 871.56], + [10000, 0.2, 0.1, 0.2543, 123.528], + [0.1, 0.2, 0.001, 0.45972, 411.124], + [75, 2, 0.0001, 0.2923, 1278.086], + [10000, 0.5, 0.01, 0.24726, 383.876], + [3000, 0.4, 0.001, 0.2471, 701.77], + [10000, 0.9, 0.1, 0.26962, 138.332], + [5, 1.5, 0.1, 0.42806, 59.038], + [400, 0.3, 0.1, 0.27886, 85.946], + [800, 0.5, 0.1, 0.27346, 88.518], + [7000, 0.1, 0.001, 0.24146, 527.12], + [900, 0.6, 0.01, 0.27454, 195.314], + [5, 1.5, 0.0001, 0.34628, 1175.276], + [0.1, 0.2, 0.0001, 0.466, 1157.236], + [0.1, 1.5, 0.001, 0.3557, 418.608], + [300, 0.6, 0.001, 0.28734, 362.69], + [600, 0.5, 0.001, 0.2763, 369.874], + [2000, 5, 0.01, 0.281225, 173.8725], + [1, 0.2, 0.01, 0.45894, 177.294], + [25, 0.9, 0.001, 0.3548, 393.792], + [5, 0.8, 0.001, 0.39164, 355.038], + [100, 0.2, 0.1, 0.38874, 80.036], + [5000, 0.2, 0.0001, 0.2438, 944.946], + [25, 0.6, 0.001, 0.37614, 413.888], + [1000, 0.6, 0.01, 0.27346, 210.89], + [300, 2, 0.1, 0.2842, 79.252], + [25, 0.9, 0.0001, 0.35434, 830.464], + [4000, 0.7, 0.001, 0.25078, 716.664], + [600, 1.5, 0.001, 0.2825, 361.05], + [1000000, 0.1, 0.001, 0.5, 124.676], + [500, 3, 0.01, 0.2841, 191.144], + [5000, 1, 0.001, 0.26034, 365.852], + [700, 5, 0.01, 0.2848, 161.416], + [50, 0.4, 0.0001, 0.36498, 1013.074], + [1000000, 0.6, 0.0001, 0.5, 918.886], + [1000000, 0.7, 0.001, 0.5, 311.996], + [2000, 0.8, 0.1, 0.26996, 103.672], + [10000, 0.4, 0.01, 0.24666, 367.468], + [5, 2, 0.1, 0.38626, 53.478], + [0.1, 0.8, 0.001, 0.40396, 401.224], + [200, 1, 0.001, 0.3005, 320.878], + [100, 0.3, 0.0001, 0.33274, 891.804], + [10, 0.3, 0.01, 0.4367, 177.636], + [1, 1, 0.01, 0.39912, 157.822], + [4000, 0.8, 0.0001, 0.25138, 1119.464], + [5000, 0.3, 0.1, 0.2609, 136.39], + [10000, 0.2, 0.01, 0.24258, 334.754], + [300, 0.1, 0.1, 0.28304, 85.45], + [700, 1, 0.001, 0.27946, 365.704], + [100, 1, 0.001, 0.32406, 412.482], + [700, 0.2, 0.1, 0.26872, 89.436], + [50, 3, 0.0001, 0.28218, 1474.37], + [4000, 5, 0.1, 0.27622, 141.548], + [600, 3, 0.001, 0.27646, 618.504], + [500, 0.3, 0.01, 0.27672, 193.994], + [5, 0.2, 0.001, 0.44494, 362.062], + [800, 2, 0.001, 0.27798, 392.928], + [25, 0.3, 0.1, 0.4656, 61.826], + [6000, 0.1, 0.01, 0.25708, 160.788], + [900, 3, 0.001, 0.26852, 684.37], + [1000000, 0.1, 0.01, 0.5, 140.52], + [10000, 2, 0.001, 0.25988, 743.122], + [10000, 0.7, 0.01, 0.24962, 393.284], + [1000000, 5, 0.1, 0.2842, 103.308], + [6000, 0.8, 0.0001, 0.25508, 1145.044], + [1000, 0.4, 0.001, 0.27102, 297.974], + [100, 0.4, 0.1, 0.36014, 81.222], + [500, 0.6, 0.01, 0.2792, 199.318], + [75, 3, 0.0001, 0.28032, 1433.972], + [500, 3, 0.0001, 0.2662, 1750.338], + [400, 0.2, 0.1, 0.27794, 88.754], + [0.1, 0.9, 0.1, 0.44702, 61.09], + [0.1, 0.4, 0.001, 0.44028, 426.974], + [10, 0.1, 0.001, 0.44644, 427.25], + [75, 1, 0.1, 0.37068, 68.896], + [4000, 0.7, 0.0001, 0.25016, 1149.51], + [800, 0.5, 0.0001, 0.2735, 710.34], + [10, 0.1, 0.01, 0.4537, 186.542], + [2000, 0.1, 0.0001, 0.244475, 1474.9], + [10000, 3, 0.1, 0.27304, 172.976], + [1, 0.7, 0.1, 0.43742, 64.206], + [10, 1.5, 0.01, 0.3467, 180.542], + [25, 0.8, 0.0001, 0.36112, 825.902], + [700, 1, 0.0001, 0.278375, 719.02], + [10, 1.5, 0.0001, 0.33882, 1194.746], + [400, 0.8, 0.1, 0.28164, 84.476], + [25, 3, 0.0001, 0.33108, 1055.664], + [300, 0.8, 0.01, 0.28716, 187.23], + [75, 1.5, 0.001, 0.34712, 300.506], + [600, 0.7, 0.1, 0.27614, 93.624], + [25, 0.9, 0.01, 0.36598, 204.464], + [4000, 3, 0.0001, 0.2653, 1488.496], + [4000, 0.4, 0.0001, 0.24662, 1030.786], + [4000, 0.5, 0.01, 0.26518, 162.112], + [7000, 5, 0.01, 0.27704, 213.324], + [200, 0.6, 0.1, 0.29356, 85.848], + [3000, 0.7, 0.01, 0.26922, 165.608], + [500, 0.1, 0.001, 0.27444, 391.212], + [100, 0.2, 0.0001, 0.33904, 866.464], + [1000, 0.3, 0.1, 0.26862, 92.152], + [1000, 0.5, 0.0001, 0.2722, 640.96], + [700, 0.3, 0.1, 0.27088, 87.974], + [0.1, 0.8, 0.01, 0.41062, 183.924], + [800, 5, 0.0001, 0.26742, 1459.934], + [100, 0.7, 0.01, 0.3199, 205.46], + [0.1, 0.9, 0.0001, 0.39922, 1159.944], + [900, 2, 0.1, 0.28026, 90.612], + [10, 0.4, 0.01, 0.42598, 178.41], + [1000, 0.3, 0.0001, 0.2694, 596.753333333], + [50, 2, 0.0001, 0.29378, 1441.402], + [1000000, 0.6, 0.001, 0.5, 320.392], + [10000, 0.5, 0.001, 0.24746, 589.576], + [200, 0.3, 0.1, 0.3012, 83.166], + [3000, 1.5, 0.1, 0.27166, 131.988], + [10, 0.1, 0.0001, 0.44892, 967.918], + [3000, 0.1, 0.1, 0.26074, 101.744], + [6000, 0.1, 0.0001, 0.24188, 912.088], + [900, 0.2, 0.001, 0.26962, 293.434], + [75, 0.9, 0.001, 0.32148, 392.108], + [25, 3, 0.001, 0.38214, 267.306], + [100, 0.8, 0.1, 0.32448, 80.456], + [500, 1.5, 0.0001, 0.27062, 1863.484], + [1, 0.9, 0.0001, 0.39268, 1032.246], + [5000, 0.9, 0.01, 0.2654, 175.872], + [1, 5, 0.1, 0.32704, 72.588], + [75, 0.5, 0.01, 0.34164, 202.41], + [900, 0.5, 0.1, 0.27102, 93.722], + [0.1, 5, 0.001, 0.29542, 397.414], + [600, 0.6, 0.1, 0.27544, 90.444], + [100, 0.3, 0.1, 0.379125, 81.1075], + [25, 2, 0.1, 0.32646, 72.75], + [3000, 0.4, 0.1, 0.2636, 109.338], + [300, 3, 0.001, 0.28696, 348.452], + [500, 2, 0.1, 0.28196, 82.392], + [50, 0.1, 0.0001, 0.39982, 1006.968], + [5, 0.5, 0.0001, 0.42106, 962.666], + [10, 5, 0.01, 0.2939, 178.594], + [6000, 1.5, 0.0001, 0.26338, 1164.17], + [200, 1, 0.1, 0.29364, 80.334], + [400, 0.8, 0.01, 0.28336, 196.466], + [800, 0.1, 0.01, 0.26832, 188.172], + [4000, 0.8, 0.01, 0.26672, 176.216], + [25, 0.6, 0.01, 0.39018, 201.18], + [500, 1, 0.01, 0.28042, 182.33], + [5, 0.1, 0.01, 0.46082, 175.868]] + + c_values = (0.10000000000000001, + 1.0, + 5.0, + 10.0, + 25.0, + 50.0, + 75.0, + 100.0, + 200.0, + 300.0, + 400.0, + 500.0, + 600.0, + 700.0, + 800.0, + 900.0, + 1000.0, + 2000.0, + 3000.0, + 4000.0, + 5000.0, + 6000.0, + 7000.0, + 10000.0, + 1000000.0) + alpha_values = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 1.5, 2, 3, 5) + epsilon_values = (0.1, 0.01, 0.001, 0.0001) + + # Convert the array into an easily accesible tree + config_tree = dict() + for C_value in c_values: + config_tree[C_value] = dict() + for alpha_value in alpha_values: + config_tree[C_value][alpha_value] = dict() + for epsilon_value in epsilon_values: + config_tree[C_value][alpha_value][epsilon_value] = [np.inf, np.inf] + + # Populate the tree + for config in svm_grid_configs: + config_tree[config[0]][config[1]][config[2]][0] = config[3] + config_tree[config[0]][config[1]][config[2]][1] = config[4] + + print c, alpha, epsilon + if ret_time: + return config_tree[c_values[c]][alpha_values[alpha]][epsilon_values[epsilon]][1] + else: + return config_tree[c_values[c]][alpha_values[alpha]][epsilon_values[epsilon]][0] diff --git a/benchmarks/svm_on_grid/config.cfg b/benchmarks/svm_on_grid/config.cfg new file mode 100644 index 00000000..1eb3085f --- /dev/null +++ b/benchmarks/svm_on_grid/config.cfg @@ -0,0 +1,13 @@ +[SMAC] +p = smac_2_06_01-dev/params.pcs + +[TPE] +space = space.py + +[SPEARMINT] +config = config.pb + +[HPOLIB] +function = python ../svm.py +number_of_jobs = 100 +result_on_terminate = 1 diff --git a/benchmarks/svm_on_grid/hyperopt_august2013_mod/random_hyperopt_august2013_mod b/benchmarks/svm_on_grid/hyperopt_august2013_mod/random_hyperopt_august2013_mod new file mode 120000 index 00000000..6b6e627b --- /dev/null +++ b/benchmarks/svm_on_grid/hyperopt_august2013_mod/random_hyperopt_august2013_mod @@ -0,0 +1 @@ +random_hyperopt_august2013_mod \ No newline at end of file diff --git a/benchmarks/svm_on_grid/hyperopt_august2013_mod/space.py b/benchmarks/svm_on_grid/hyperopt_august2013_mod/space.py new file mode 100644 index 00000000..f2a2a583 --- /dev/null +++ b/benchmarks/svm_on_grid/hyperopt_august2013_mod/space.py @@ -0,0 +1,5 @@ +from hyperopt import hp + +space = {'C': hp.quniform('C', 0, 24, 1), + 'alpha': hp.quniform('alpha', 0, 13, 1), + 'epsilon': hp.quniform('epsilon', 0, 3, 1)} diff --git a/benchmarks/svm_on_grid/random_hyperopt_august2013_mod b/benchmarks/svm_on_grid/random_hyperopt_august2013_mod new file mode 120000 index 00000000..c4f9ce32 --- /dev/null +++ b/benchmarks/svm_on_grid/random_hyperopt_august2013_mod @@ -0,0 +1 @@ +hyperopt_august2013_mod \ No newline at end of file diff --git a/benchmarks/svm_on_grid/smac_2_06_01-dev/params.pcs b/benchmarks/svm_on_grid/smac_2_06_01-dev/params.pcs new file mode 100644 index 00000000..bfd0367a --- /dev/null +++ b/benchmarks/svm_on_grid/smac_2_06_01-dev/params.pcs @@ -0,0 +1,4 @@ +C [0, 24] [0]i +alpha [0, 13] [0]i +epsilon [0, 3] [0]i + diff --git a/benchmarks/svm_on_grid/spearmint_april2013_mod/config.pb b/benchmarks/svm_on_grid/spearmint_april2013_mod/config.pb new file mode 100644 index 00000000..2df3dff7 --- /dev/null +++ b/benchmarks/svm_on_grid/spearmint_april2013_mod/config.pb @@ -0,0 +1,27 @@ +language: PYTHON +name: "HPOlib.cv" + +variable { + name: "C" + type: INT + size: 1 + min: 0 + max: 24 +} + +variable { + name: "alpha" + type: INT + size: 1 + min: 0 + max: 13 +} + +variable { + name: "epsilon" + type: INT + size: 1 + min: 0 + max: 3 +} + diff --git a/benchmarks/svm_on_grid/svm.py b/benchmarks/svm_on_grid/svm.py new file mode 100644 index 00000000..61e3100b --- /dev/null +++ b/benchmarks/svm_on_grid/svm.py @@ -0,0 +1,41 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import time + +import HPOlib.benchmark_util as benchmark_util +import HPOlib.benchmark_functions as benchmark_functions + +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" +__credits__ = ["Jasper Snoek", "Ryan P. Adams", "Hugo Larochelle"] + + +def main(params, ret_time=False, **kwargs): + print 'Params: ', params, '\n' + y = benchmark_functions.save_svm_on_grid(params, opt_time=ret_time, **kwargs) + print 'Result: ', y + return y + + +if __name__ == "__main__": + args, cli_params = benchmark_util.parse_cli() + result = main(cli_params, ret_time=False, **args) + duration = main(cli_params, ret_time=True, **args) + print "Result for ParamILS: %s, %f, 1, %f, %d, %s" % \ + ("SAT", abs(duration), result, -1, str(__file__)) \ No newline at end of file From 55e6be181966d02cca842489c91ede6373195dd1 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 21 Mar 2014 18:00:17 +0100 Subject: [PATCH 123/139] FIX: LaTeX table template --- HPOlib/Plotting/generateTexTable.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/HPOlib/Plotting/generateTexTable.py b/HPOlib/Plotting/generateTexTable.py index 47e27ff7..7348530f 100644 --- a/HPOlib/Plotting/generateTexTable.py +++ b/HPOlib/Plotting/generateTexTable.py @@ -37,8 +37,9 @@ template_string = \ -"""\\documentclass{article} % For LaTeX2 +"""\\documentclass[landscape]{article} % For LaTeX2 +\\usepackage[landscape]{geometry} \\usepackage{multirow} % import command \multicolmun \\usepackage{tabularx} % Convenient table formatting \\usepackage{booktabs} % provides \toprule, \midrule and \bottomrule @@ -46,7 +47,7 @@ \\begin{document} \\begin{table}[t] -\\begin{tabularx}{\\textwidth}{lr|Xr|Xr|Xr} +\\begin{tabularx}{\\textwidth}{lr{%- for name in result_values -%}|Xr{%- endfor -%}} \\toprule \multicolumn{2}{l}{} {%- for name in result_values -%} @@ -62,7 +63,7 @@ {{ experiment }} & {{ evals }} {%- for name in result_values -%} {%- set results = result_values[name] -%} -{{ ' & ' }}{{ results['mean'] }}$\\pm${{ results['std'] }} & {{ results['min'] }} +{{ ' & ' }}{{ results['mean']|round(3, 'floor') }}$\\pm${{ results['std']|round(3, 'floor')}} & {{ results['min']|round(3, 'floor') }} {%- endfor %} \\\\ \\bottomrule \\end{tabularx} From cf6ff2246f09c8f73377690a61ebd1759ee28d1a Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Fri, 21 Mar 2014 18:26:45 +0100 Subject: [PATCH 124/139] ENHANCEMENT: #39 Adding camelback function as ruby benchmark --- benchmarks/camelback/__init__.py | 2 + benchmarks/camelback/camelback.rb | 46 +++++++++++++++++++ benchmarks/camelback/config.cfg | 13 ++++++ .../hyperopt_august2013_mod/space.py | 4 ++ .../camelback/random_hyperopt_august2013_mod | 1 + .../camelback/smac_2_06_01-dev/params.pcs | 2 + .../spearmint_april2013_mod/config.pb | 18 ++++++++ 7 files changed, 86 insertions(+) create mode 100644 benchmarks/camelback/__init__.py create mode 100644 benchmarks/camelback/camelback.rb create mode 100644 benchmarks/camelback/config.cfg create mode 100644 benchmarks/camelback/hyperopt_august2013_mod/space.py create mode 120000 benchmarks/camelback/random_hyperopt_august2013_mod create mode 100644 benchmarks/camelback/smac_2_06_01-dev/params.pcs create mode 100644 benchmarks/camelback/spearmint_april2013_mod/config.pb diff --git a/benchmarks/camelback/__init__.py b/benchmarks/camelback/__init__.py new file mode 100644 index 00000000..24cd9aaa --- /dev/null +++ b/benchmarks/camelback/__init__.py @@ -0,0 +1,2 @@ +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" \ No newline at end of file diff --git a/benchmarks/camelback/camelback.rb b/benchmarks/camelback/camelback.rb new file mode 100644 index 00000000..d597d1a5 --- /dev/null +++ b/benchmarks/camelback/camelback.rb @@ -0,0 +1,46 @@ +#! /usr/bin/ruby + +# ruby camelback.rb --fold 2 --folds 20 --params -x -0.0898 -y 0.7126 +# >> Result for ParamILS: SAT, 0.000095, 1, -1.031628, -1, camelback.rb +# -2 < x < 2, -1 < y < 1 +# 2 Global minima: f(x) = -1.0316 (0.0898, -0.7126), (-0.0898, 0.7126) + +def camelback(x, y) + puts 'Params: ', x, y + + tmp1 = (4 - 2.1 * x**2 + (x**4)/3) * x**2 + tmp2 = x*y + tmp3 = (-4 + 4 * y**2) * y**2 + y = tmp1 + tmp2 + tmp3 + + puts 'Result: ', y + return y +end + +starttime = Time.now() +x = -1 +y = -1 +puts ARGV +if ARGV[5] == '-y' + y = ARGV[6].to_f +elsif ARGV[5] == '-x' + x = ARGV[6].to_f +else + + abort("Result for ParamILS: CRASH, 1, 1, 10, -1, camelback.rb") +end + +if ARGV[7] == '-y' + y = ARGV[8].to_f +elsif ARGV[7] == '-x' + x = ARGV[8].to_f +else + abort("Result for ParamILS: CRASH, 1, 1, 10, -1, camelback.rb") +end + +result = camelback(x, y) + +# Benchmark need to take some time, because of a bug in runsolver +sleep(1) +duration = Time.now() - starttime +printf "\nResult for ParamILS: %s, %f, 1, %f, %d, %s\n", "SAT", duration, result, -1, "camelback.rb" \ No newline at end of file diff --git a/benchmarks/camelback/config.cfg b/benchmarks/camelback/config.cfg new file mode 100644 index 00000000..2cb8de8e --- /dev/null +++ b/benchmarks/camelback/config.cfg @@ -0,0 +1,13 @@ +[SMAC] +p = smac_2_06_01-dev/params.pcs + +[TPE] +space = space.py + +[SPEARMINT] +config = config.pb + +[HPOLIB] +function = ruby ../camelback.rb +number_of_jobs = 100 +result_on_terminate = 10 diff --git a/benchmarks/camelback/hyperopt_august2013_mod/space.py b/benchmarks/camelback/hyperopt_august2013_mod/space.py new file mode 100644 index 00000000..3a42ba46 --- /dev/null +++ b/benchmarks/camelback/hyperopt_august2013_mod/space.py @@ -0,0 +1,4 @@ +from hyperopt import hp + +space = {'x': hp.uniform('x', -2, 2), + 'y': hp.uniform('y', -1, 1)} \ No newline at end of file diff --git a/benchmarks/camelback/random_hyperopt_august2013_mod b/benchmarks/camelback/random_hyperopt_august2013_mod new file mode 120000 index 00000000..398d85cc --- /dev/null +++ b/benchmarks/camelback/random_hyperopt_august2013_mod @@ -0,0 +1 @@ +hyperopt_august2013_mod/ \ No newline at end of file diff --git a/benchmarks/camelback/smac_2_06_01-dev/params.pcs b/benchmarks/camelback/smac_2_06_01-dev/params.pcs new file mode 100644 index 00000000..d42b73a6 --- /dev/null +++ b/benchmarks/camelback/smac_2_06_01-dev/params.pcs @@ -0,0 +1,2 @@ +x [-2, 2] [0] +y [-1, 1] [0] diff --git a/benchmarks/camelback/spearmint_april2013_mod/config.pb b/benchmarks/camelback/spearmint_april2013_mod/config.pb new file mode 100644 index 00000000..13648b4a --- /dev/null +++ b/benchmarks/camelback/spearmint_april2013_mod/config.pb @@ -0,0 +1,18 @@ +language: PYTHON +name: "HPOlib.cv" + +variable { + name: "x" + type: FLOAT + size: 1 + min: -2 + max: 2 +} + +variable { + name: "y" + type: FLOAT + size: 1 + min: -1 + max: 1 +} From ac67ef44b20174328527e4697fcc006689f848af Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Sun, 23 Mar 2014 16:35:40 +0100 Subject: [PATCH 125/139] ENHANCEMENT: #36 HPOlib-convert add script HPOlib-convert to convert search spaces from TPE -> SMAC TPE -> SPEARMINT SMAC -> SPEARMINT --- HPOlib/format_converter/__init__.py | 1 + HPOlib/format_converter/convert.py | 135 ++++++ ...{SMACSpearmint.py => smac_to_spearmint.py} | 24 +- .../spearmint_april2013_mod_spearmint_pb2.py | 406 ++++++++++++++++++ .../{TpeSMAC.py => tpe_to_smac.py} | 45 +- scripts/HPOlib-convert | 26 ++ setup.py | 8 +- 7 files changed, 602 insertions(+), 43 deletions(-) create mode 100644 HPOlib/format_converter/__init__.py create mode 100644 HPOlib/format_converter/convert.py rename HPOlib/format_converter/{SMACSpearmint.py => smac_to_spearmint.py} (84%) create mode 100644 HPOlib/format_converter/spearmint_april2013_mod_spearmint_pb2.py rename HPOlib/format_converter/{TpeSMAC.py => tpe_to_smac.py} (87%) create mode 100644 scripts/HPOlib-convert diff --git a/HPOlib/format_converter/__init__.py b/HPOlib/format_converter/__init__.py new file mode 100644 index 00000000..163eead8 --- /dev/null +++ b/HPOlib/format_converter/__init__.py @@ -0,0 +1 @@ +__author__ = 'eggenspk' diff --git a/HPOlib/format_converter/convert.py b/HPOlib/format_converter/convert.py new file mode 100644 index 00000000..7233767e --- /dev/null +++ b/HPOlib/format_converter/convert.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python + +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from argparse import ArgumentParser +import os +from string import upper +import sys +import tempfile + +import smac_to_spearmint +import tpe_to_smac + + +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" + + +def smac_to_spearmint_helper(space, save=""): + # print "Convert %s from SMAC to SPEARMINT" % space + return smac_to_spearmint.convert_smac_to_spearmint(space) + + +def smac_to_tpe_helper(space, save=""): + print "This is not yet implemented" + + +def spearmint_to_smac_helper(space, save=""): + print "This is not yet implemented" + + +def spearmint_to_tpe_helper(space, save=""): + print "This is not yet implemented" + + +def tpe_to_spearmint_helper(space, save=""): + try: + import hyperopt + except ImportError: + print "Cannot find hyperopt. To use this converter, modify $PYTHONPATH to contain a hyperopt installation" + + # First convert to smac + tmp = tpe_to_smac.convert_tpe_to_smac_from_file(space) + handle, tmp_file_name = tempfile.mkstemp() + fh = open(tmp_file_name, 'w') + fh.write(tmp) + fh.close() + + # From smac convert to spearmint + new_space = smac_to_spearmint.convert_smac_to_spearmint(tmp_file_name) + + os.remove(tmp_file_name) + return new_space + + +def tpe_to_smac_helper(space, save=""): + try: + import hyperopt + except ImportError: + print "Cannot find hyperopt. To use this converter, modify $PYTHONPATH to contain a hyperopt installation" + return tpe_to_smac.convert_tpe_to_smac_from_file(space) + + +def main(): + # python convert.py --from SMAC --to TPE -f space.any -s space.else + prog = "python convert.py" + description = "Automatically convert a searchspace from one format to another" + + parser = ArgumentParser(description=description, prog=prog) + + parser.add_argument("--from", dest="conv_from", choices=['SMAC', 'Smac', 'smac', + 'TPE', 'Tpe', 'tpe', 'hyperopt', + 'SPEARMINT', 'Spearmint', 'spearmint'], + default="", help="Convert from which format?", required=True) + parser.add_argument("--to", dest="conv_to", choices=['SMAC', 'Smac', 'smac', + 'TPE', 'Tpe', 'tpe', 'hyperopt', + 'SPEARMINT', 'Spearmint', 'spearmint'], + default="", help="Convert to which format?", required=True) + parser.add_argument("-f", "--file", dest="space", + default="", help="Where is the searchspace to be converted?", required=True) + parser.add_argument("-s", "--save", dest="save", + default="", help="Where to save the new searchspace?") + + args, unknown = parser.parse_known_args() + + space = os.path.abspath(args.space) + if not os.path.isfile(space): + print "%s is not a valid path" % space + sys.exit(1) + + # Unifying strings + args.conv_to = upper(args.conv_to) + args.conv_from = upper(args.conv_from) + if args.conv_from == "HYPEROPT": + args.conv_from = "TPE" + if args.conv_to == "HYPEROPT": + args.conv_to == "TPE" + + if args.conv_to == args.conv_from: + print "Converting from %s to %s makes no sense" % (args.conv_to, args.conv_from) + + # This is like a switch statement + options = {'SMAC': {'SPEARMINT': smac_to_spearmint_helper, + 'TPE': smac_to_tpe_helper}, + 'SPEARMINT': {'SMAC': spearmint_to_smac_helper, + 'TPE': spearmint_to_tpe_helper}, + 'TPE': {'SPEARMINT': tpe_to_spearmint_helper, + 'SMAC': tpe_to_smac_helper} + } + new_space = options[args.conv_from][args.conv_to](space, args.save) + if args.save != "": + fh = open(args.save, 'w') + fh.write(new_space) + fh.close() + else: + print new_space + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/HPOlib/format_converter/SMACSpearmint.py b/HPOlib/format_converter/smac_to_spearmint.py similarity index 84% rename from HPOlib/format_converter/SMACSpearmint.py rename to HPOlib/format_converter/smac_to_spearmint.py index a71b051a..8c5e753e 100644 --- a/HPOlib/format_converter/SMACSpearmint.py +++ b/HPOlib/format_converter/smac_to_spearmint.py @@ -16,26 +16,25 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/usr/bin/env python """ -This script reads a SMAC search space and prints out an spearmint searchspace +This method reads a SMAC search space and returns a spearmint searchspace Default value and conditions are lost """ import sys import os -path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../optimizers/") -sys.path.append(path) -from spearmint_april2013_mod.spearmint_pb2 import Experiment, PYTHON + +from spearmint_april2013_mod_spearmint_pb2 import Experiment, PYTHON from google.protobuf import text_format import numpy as np -def convertSMACToSpearmint(filename): - file = open(filename) + +def convert_smac_to_spearmint(filename): + fl_handle = open(filename) params = [] - for line in file: + for line in fl_handle: line = line.strip() if line.strip().find('#') == 0 or line == "": continue @@ -97,11 +96,6 @@ def convertSMACToSpearmint(filename): exp.language = PYTHON exp.name = "cv" exp.variable.extend(params) - print text_format.MessageToString(exp) - -if __name__ == '__main__': - if len(sys.argv) != 2: - sys.stdout.write("Wrong number of Arguments\nUsage: python SMACSpearmint.py ") - sys.exit(1) - convertSMACToSpearmint(sys.argv[1]) + return text_format.MessageToString(exp) + diff --git a/HPOlib/format_converter/spearmint_april2013_mod_spearmint_pb2.py b/HPOlib/format_converter/spearmint_april2013_mod_spearmint_pb2.py new file mode 100644 index 00000000..f215e531 --- /dev/null +++ b/HPOlib/format_converter/spearmint_april2013_mod_spearmint_pb2.py @@ -0,0 +1,406 @@ +## +# Copyright (C) 2012 Jasper Snoek, Hugo Larochelle and Ryan P. Adams +# +# This code is written for research and educational purposes only to +# supplement the paper entitled +# "Practical Bayesian Optimization of Machine Learning Algorithms" +# by Snoek, Larochelle and Adams +# Advances in Neural Information Processing Systems, 2012 +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +## +# Copyright (C) 2012 Jasper Snoek, Hugo Larochelle and Ryan P. Adams +# +# This code is written for research and educational purposes only to +# supplement the paper entitled +# "Practical Bayesian Optimization of Machine Learning Algorithms" +# by Snoek, Larochelle and Adams +# Advances in Neural Information Processing Systems, 2012 +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# Generated by the protocol buffer compiler. DO NOT EDIT! + +from google.protobuf import descriptor +from google.protobuf import message +from google.protobuf import reflection +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + + + +DESCRIPTOR = descriptor.FileDescriptor( + name='spearmint.proto', + package='', + serialized_pb='\n\x0fspearmint.proto\"\xcc\x01\n\x03Job\x12\n\n\x02id\x18\x01 \x02(\x04\x12\x10\n\x08\x65xpt_dir\x18\x02 \x02(\t\x12\x0c\n\x04name\x18\x03 \x02(\t\x12\x1b\n\x08language\x18\x04 \x02(\x0e\x32\t.Language\x12\x0e\n\x06status\x18\x05 \x01(\t\x12\x19\n\x05param\x18\x06 \x03(\x0b\x32\n.Parameter\x12\x10\n\x08submit_t\x18\x07 \x01(\x04\x12\x0f\n\x07start_t\x18\x08 \x01(\x04\x12\r\n\x05\x65nd_t\x18\t \x01(\x04\x12\r\n\x05value\x18\n \x01(\x01\x12\x10\n\x08\x64uration\x18\x0b \x01(\x01\"L\n\tParameter\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\x0f\n\x07int_val\x18\x02 \x03(\x03\x12\x0f\n\x07str_val\x18\x03 \x03(\t\x12\x0f\n\x07\x64\x62l_val\x18\x04 \x03(\x01\"\x91\x02\n\nExperiment\x12\x1b\n\x08language\x18\x01 \x02(\x0e\x32\t.Language\x12\x0c\n\x04name\x18\x02 \x02(\t\x12+\n\x08variable\x18\x03 \x03(\x0b\x32\x19.Experiment.ParameterSpec\x1a\xaa\x01\n\rParameterSpec\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\x0c\n\x04size\x18\x02 \x02(\r\x12,\n\x04type\x18\x03 \x02(\x0e\x32\x1e.Experiment.ParameterSpec.Type\x12\x0f\n\x07options\x18\x04 \x03(\t\x12\x0b\n\x03min\x18\x05 \x01(\x01\x12\x0b\n\x03max\x18\x06 \x01(\x01\"$\n\x04Type\x12\x07\n\x03INT\x10\x01\x12\t\n\x05\x46LOAT\x10\x02\x12\x08\n\x04\x45NUM\x10\x03*6\n\x08Language\x12\n\n\x06MATLAB\x10\x01\x12\n\n\x06PYTHON\x10\x02\x12\t\n\x05SHELL\x10\x03\x12\x07\n\x03MCR\x10\x04') + +_LANGUAGE = descriptor.EnumDescriptor( + name='Language', + full_name='Language', + filename=None, + file=DESCRIPTOR, + values=[ + descriptor.EnumValueDescriptor( + name='MATLAB', index=0, number=1, + options=None, + type=None), + descriptor.EnumValueDescriptor( + name='PYTHON', index=1, number=2, + options=None, + type=None), + descriptor.EnumValueDescriptor( + name='SHELL', index=2, number=3, + options=None, + type=None), + descriptor.EnumValueDescriptor( + name='MCR', index=3, number=4, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=580, + serialized_end=634, +) + + +MATLAB = 1 +PYTHON = 2 +SHELL = 3 +MCR = 4 + + +_EXPERIMENT_PARAMETERSPEC_TYPE = descriptor.EnumDescriptor( + name='Type', + full_name='Experiment.ParameterSpec.Type', + filename=None, + file=DESCRIPTOR, + values=[ + descriptor.EnumValueDescriptor( + name='INT', index=0, number=1, + options=None, + type=None), + descriptor.EnumValueDescriptor( + name='FLOAT', index=1, number=2, + options=None, + type=None), + descriptor.EnumValueDescriptor( + name='ENUM', index=2, number=3, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=542, + serialized_end=578, +) + + +_JOB = descriptor.Descriptor( + name='Job', + full_name='Job', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + descriptor.FieldDescriptor( + name='id', full_name='Job.id', index=0, + number=1, type=4, cpp_type=4, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='expt_dir', full_name='Job.expt_dir', index=1, + number=2, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='name', full_name='Job.name', index=2, + number=3, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='language', full_name='Job.language', index=3, + number=4, type=14, cpp_type=8, label=2, + has_default_value=False, default_value=1, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='status', full_name='Job.status', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='param', full_name='Job.param', index=5, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='submit_t', full_name='Job.submit_t', index=6, + number=7, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='start_t', full_name='Job.start_t', index=7, + number=8, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='end_t', full_name='Job.end_t', index=8, + number=9, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='value', full_name='Job.value', index=9, + number=10, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='duration', full_name='Job.duration', index=10, + number=11, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=20, + serialized_end=224, +) + + +_PARAMETER = descriptor.Descriptor( + name='Parameter', + full_name='Parameter', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + descriptor.FieldDescriptor( + name='name', full_name='Parameter.name', index=0, + number=1, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='int_val', full_name='Parameter.int_val', index=1, + number=2, type=3, cpp_type=2, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='str_val', full_name='Parameter.str_val', index=2, + number=3, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='dbl_val', full_name='Parameter.dbl_val', index=3, + number=4, type=1, cpp_type=5, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=226, + serialized_end=302, +) + + +_EXPERIMENT_PARAMETERSPEC = descriptor.Descriptor( + name='ParameterSpec', + full_name='Experiment.ParameterSpec', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + descriptor.FieldDescriptor( + name='name', full_name='Experiment.ParameterSpec.name', index=0, + number=1, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='size', full_name='Experiment.ParameterSpec.size', index=1, + number=2, type=13, cpp_type=3, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='type', full_name='Experiment.ParameterSpec.type', index=2, + number=3, type=14, cpp_type=8, label=2, + has_default_value=False, default_value=1, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='options', full_name='Experiment.ParameterSpec.options', index=3, + number=4, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='min', full_name='Experiment.ParameterSpec.min', index=4, + number=5, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='max', full_name='Experiment.ParameterSpec.max', index=5, + number=6, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _EXPERIMENT_PARAMETERSPEC_TYPE, + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=408, + serialized_end=578, +) + +_EXPERIMENT = descriptor.Descriptor( + name='Experiment', + full_name='Experiment', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + descriptor.FieldDescriptor( + name='language', full_name='Experiment.language', index=0, + number=1, type=14, cpp_type=8, label=2, + has_default_value=False, default_value=1, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='name', full_name='Experiment.name', index=1, + number=2, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='variable', full_name='Experiment.variable', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_EXPERIMENT_PARAMETERSPEC, ], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=305, + serialized_end=578, +) + +_JOB.fields_by_name['language'].enum_type = _LANGUAGE +_JOB.fields_by_name['param'].message_type = _PARAMETER +_EXPERIMENT_PARAMETERSPEC.fields_by_name['type'].enum_type = _EXPERIMENT_PARAMETERSPEC_TYPE +_EXPERIMENT_PARAMETERSPEC.containing_type = _EXPERIMENT; +_EXPERIMENT_PARAMETERSPEC_TYPE.containing_type = _EXPERIMENT_PARAMETERSPEC; +_EXPERIMENT.fields_by_name['language'].enum_type = _LANGUAGE +_EXPERIMENT.fields_by_name['variable'].message_type = _EXPERIMENT_PARAMETERSPEC +DESCRIPTOR.message_types_by_name['Job'] = _JOB +DESCRIPTOR.message_types_by_name['Parameter'] = _PARAMETER +DESCRIPTOR.message_types_by_name['Experiment'] = _EXPERIMENT + +class Job(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType + DESCRIPTOR = _JOB + + # @@protoc_insertion_point(class_scope:Job) + +class Parameter(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType + DESCRIPTOR = _PARAMETER + + # @@protoc_insertion_point(class_scope:Parameter) + +class Experiment(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType + + class ParameterSpec(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType + DESCRIPTOR = _EXPERIMENT_PARAMETERSPEC + + # @@protoc_insertion_point(class_scope:Experiment.ParameterSpec) + DESCRIPTOR = _EXPERIMENT + + # @@protoc_insertion_point(class_scope:Experiment) + +# @@protoc_insertion_point(module_scope) diff --git a/HPOlib/format_converter/TpeSMAC.py b/HPOlib/format_converter/tpe_to_smac.py similarity index 87% rename from HPOlib/format_converter/TpeSMAC.py rename to HPOlib/format_converter/tpe_to_smac.py index df3cefa7..b1317448 100644 --- a/HPOlib/format_converter/TpeSMAC.py +++ b/HPOlib/format_converter/tpe_to_smac.py @@ -24,6 +24,8 @@ """ +from hyperopt.pyll import base as base + """ Partly taken from https://github.com/jaberg/hyperopt/blob/smacout/hyperopt/smac.py """ @@ -33,8 +35,6 @@ from functools import partial -import hyperopt - class Cond(object): def __init__(self, name, val, op): @@ -60,7 +60,7 @@ def __repr__(self): def expr_to_config(expr, conditions, hps): if conditions is None: conditions = () - assert isinstance(expr, hyperopt.pyll.base.Apply) + assert isinstance(expr, base.Apply) if expr.name == 'switch': idx = expr.inputs()[0] options = expr.inputs()[1:] @@ -87,7 +87,7 @@ def expr_to_config(expr, conditions, hps): expr_to_config(ii, conditions, hps) -def expr_to_smac_config(filename): +def convert_tpe_to_smac_from_file(filename): space_name, ext = os.path.splitext(os.path.basename(filename)) #noinspection PyBroadException @@ -102,29 +102,30 @@ def expr_to_smac_config(filename): search_space = space.space - expr = hyperopt.pyll.as_apply(search_space) + expr = base.as_apply(search_space) hps = {} expr_to_config(expr, (True,), hps) + new_space = "" # print expr for label, dct in hps.items(): if dct['node'].name == "randint": assert len(dct['node'].inputs()) == 1 #randint['x', 5] -> x [0, 4]i [0] upper = dct['node'].inputs()[0].eval() - print '%s {%s} [0]' % \ + new_space += '%s {%s} [0]\n' % \ (label, ", ".join(["%s" % (i,) for i in range(upper)])) elif dct['node'].name == "categorical": # No difference to a randint node upper = dct['node'].inputs()[1].eval() - print '%s {%s} [0]' % \ - (label, ", ".join(["%s" % (i,) for i in range(upper)])) + new_space += '%s {%s} [0]\n' % \ + (label, ", ".join(["%s" % (i,) for i in range(upper)])) elif dct['node'].name == "uniform": assert len(dct['node'].inputs()) == 2 lower = dct['node'].inputs()[0].eval() upper = dct['node'].inputs()[1].eval() default = (lower+upper)/2.0 - print '%s [%s, %s] [%s]' % (label, lower, upper, default) + new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, default) elif dct['node'].name == "quniform": # Assumption: q-value is always the last value assert len(dct['node'].inputs()) == 3 @@ -133,14 +134,14 @@ def expr_to_smac_config(filename): q = dct['node'].inputs()[2].eval() default = (lower+upper)/2.0 label = "Q%s_%s" % (q, label) - print '%s [%s, %s] [%s]' % (label, lower, upper, default) + new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, default) elif dct['node'].name == "loguniform": assert len(dct['node'].inputs()) == 2 lower = dct['node'].inputs()[0].eval() upper = dct['node'].inputs()[1].eval() default = (lower+upper)/2.0 label = "LOG_%s" % (label,) - print '%s [%s, %s] [%s]' % (label, lower, upper, default) + new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, default) elif dct['node'].name == "qloguniform": assert len(dct['node'].inputs()) == 3 lower = dct['node'].inputs()[0].eval() @@ -148,14 +149,14 @@ def expr_to_smac_config(filename): q = dct['node'].inputs()[2].eval() default = (lower+upper)/2.0 label = "LOG_Q%s_%s" % (q, label) - print '%s [%s, %s] [%s]' % (label, lower, upper, default) + new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, default) elif dct['node'].name == "normal": assert len(dct['node'].inputs()) == 2 mu = dct['node'].inputs()[0].eval() sigma = dct['node'].inputs()[1].eval() lower = mu-3*sigma upper = mu+3*sigma - print '%s [%s, %s] [%s]' % (label, lower, upper, mu) + new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, mu) elif dct['node'].name == "qnormal": assert len(dct['node'].inputs()) == 3 mu = dct['node'].inputs()[0].eval() @@ -164,7 +165,7 @@ def expr_to_smac_config(filename): upper = mu+3*sigma q = dct['node'].inputs()[2].eval() label = "Q%s_%s" % (q, label) - print '%s [%s, %s] [%s]' % (label, lower, upper, mu) + new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, mu) elif dct['node'].name == "lognormal": assert len(dct['node'].inputs()) == 2 mu = dct['node'].inputs()[0].eval() @@ -172,7 +173,7 @@ def expr_to_smac_config(filename): lower = mu-3*sigma upper = mu+3*sigma label = "LOG_%s" % (label, ) - print '%s [%s, %s] [%s]' % (label, lower, upper, mu) + new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, mu) elif dct['node'].name == "qlognormal": assert len(dct['node'].inputs()) == 3 mu = dct['node'].inputs()[0].eval() @@ -181,7 +182,7 @@ def expr_to_smac_config(filename): upper = mu+3*sigma q = dct['node'].inputs()[2].eval() label = "Q%s_LOG_%s" % (q, label) - print '%s [%s, %s] [%s]' % (label, lower, upper, mu) + new_space += '%s [%s, %s] [%s]\n' % (label, lower, upper, mu) else: raise Exception("Node name %s not known" % dct['node'].name) @@ -224,14 +225,8 @@ def expr_to_smac_config(filename): # Start printing conditions # print "CCC", varying_param, varying_param_name if varying_param_name != "": - print '%s | %s in {%s}' % (label, varying_param_name, ",".join(str(i) for i in varying_param)) + new_space += '%s | %s in {%s}\n' % (label, varying_param_name, ",".join(str(i) for i in varying_param)) for key in condict.keys(): if key != varying_param_name: - print '%s | %s in {%s}' % (label, key, condict[key]) - - -if __name__ == '__main__': - if len(sys.argv) != 2: - sys.stdout.write("Wrong number of Arguments\nUsage: python TpeSMAC.py \n") - sys.exit(1) - expr_to_smac_config(sys.argv[1]) + new_space += '%s | %s in {%s}\n' % (label, key, condict[key]) + return new_space \ No newline at end of file diff --git a/scripts/HPOlib-convert b/scripts/HPOlib-convert new file mode 100644 index 00000000..94f2a43c --- /dev/null +++ b/scripts/HPOlib-convert @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" + +from HPOlib.format_converter import convert + +convert.main() \ No newline at end of file diff --git a/setup.py b/setup.py index 92216831..bfb7ba46 100644 --- a/setup.py +++ b/setup.py @@ -16,11 +16,12 @@ package_dir = {'HPOlib': 'HPOlib', 'HPOlib.config_parser': 'HPOlib/config_parser', - 'HPOlib.Plotting': 'HPOlib/Plotting'} + 'HPOlib.Plotting': 'HPOlib/Plotting', + 'HPOlib.format_converter': 'HPOlib/format_converter'} package_data = {'HPOlib.config_parser': ['*.cfg']} data_files = [] -scripts = ['scripts/HPOlib-run', 'scripts/HPOlib-plot', 'runsolver/src/runsolver'] +scripts = ['scripts/HPOlib-run', 'scripts/HPOlib-plot', 'runsolver/src/runsolver', 'scripts/HPOlib-convert'] def read(fname): @@ -30,7 +31,8 @@ def read(fname): def get_find_packages(): packages = ['HPOlib', 'HPOlib.config_parser', - 'HPOlib.Plotting'] + 'HPOlib.Plotting', + 'HPOlib.format_converter'] return packages From 9bf0328e1d7390637fe5792883a7e676c73ad965 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Mon, 24 Mar 2014 10:50:40 +0100 Subject: [PATCH 126/139] REFACTOR: Adding the cut argument to all plot scripts --- HPOlib/Plotting/doAllPlots.py | 53 ++++++++++++--------- HPOlib/Plotting/generateTexTable.py | 2 +- HPOlib/Plotting/plotBoxWhisker.py | 11 +++-- HPOlib/Plotting/plotOptimizerOverhead.py | 33 ++++++++----- HPOlib/Plotting/plotTrace.py | 29 ++++++----- HPOlib/Plotting/plotTraceWithStd_perEval.py | 5 +- HPOlib/Plotting/plotTraceWithStd_perTime.py | 3 +- HPOlib/Plotting/plot_util.py | 16 +++---- HPOlib/Plotting/statistics.py | 9 ++-- 9 files changed, 96 insertions(+), 65 deletions(-) diff --git a/HPOlib/Plotting/doAllPlots.py b/HPOlib/Plotting/doAllPlots.py index b5a1c84b..deb33406 100644 --- a/HPOlib/Plotting/doAllPlots.py +++ b/HPOlib/Plotting/doAllPlots.py @@ -32,7 +32,7 @@ __contact__ = "automl.org" -def _plot_trace(pkl_list, name_list, save="", log=False): +def _plot_trace(pkl_list, name_list, save="", cut=sys.maxint, log=False): # We have one pkl per experiment plotting_dir = os.path.dirname(os.path.realpath(__file__)) @@ -42,7 +42,7 @@ def _plot_trace(pkl_list, name_list, save="", log=False): try: os.chdir(plotting_dir) import plotTrace - plotTrace.main(pkl_list, name_list, save=save, log=log) + plotTrace.main(pkl_list, name_list, save=save, log=log, cut=cut) os.chdir(cur_dir) sys.stdout.write("passed\n") except Exception, e: @@ -50,7 +50,8 @@ def _plot_trace(pkl_list, name_list, save="", log=False): sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) -def _trace_with_std_per_eval(pkl_list, name_list, save="", log=False): +def _trace_with_std_per_eval(pkl_list, name_list, save="", log=False, + cut=sys.maxint): plotting_dir = os.path.dirname(os.path.realpath(__file__)) cur_dir = os.getcwd() @@ -59,7 +60,7 @@ def _trace_with_std_per_eval(pkl_list, name_list, save="", log=False): os.chdir(plotting_dir) import plotTraceWithStd_perEval plotTraceWithStd_perEval.main(pkl_list, name_list, autofill=True, - optimum=0, save=save, log=log) + optimum=0, save=save, log=log, cut=cut) os.chdir(cur_dir) sys.stdout.write("passed\n") except Exception, e: @@ -67,7 +68,8 @@ def _trace_with_std_per_eval(pkl_list, name_list, save="", log=False): sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) -def _trace_with_std_per_time(pkl_list, name_list, save="", log=False): +def _trace_with_std_per_time(pkl_list, name_list, save="", log=False, + cut=sys.maxint): plotting_dir = os.path.dirname(os.path.realpath(__file__)) cur_dir = os.getcwd() @@ -76,7 +78,7 @@ def _trace_with_std_per_time(pkl_list, name_list, save="", log=False): os.chdir(plotting_dir) import plotTraceWithStd_perTime plotTraceWithStd_perTime.main(pkl_list, name_list, autofill=True, - optimum=0, save=save, log=log) + optimum=0, save=save, log=log, cut=cut) os.chdir(cur_dir) sys.stdout.write("passed\n") except Exception, e: @@ -84,7 +86,7 @@ def _trace_with_std_per_time(pkl_list, name_list, save="", log=False): sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) -def _optimizer_overhead(pkl_list, name_list, save, log): +def _optimizer_overhead(pkl_list, name_list, save, log, cut=sys.maxint): plotting_dir = os.path.dirname(os.path.realpath(__file__)) cur_dir = os.getcwd() @@ -93,7 +95,7 @@ def _optimizer_overhead(pkl_list, name_list, save, log): os.chdir(plotting_dir) import plotOptimizerOverhead plotOptimizerOverhead.main(pkl_list, name_list, autofill=True, - log=log, save=save) + log=log, save=save, cut=cut) os.chdir(cur_dir) sys.stdout.write("passed\n") except Exception, e: @@ -101,7 +103,7 @@ def _optimizer_overhead(pkl_list, name_list, save, log): sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) -def _box_whisker(pkl_list, name_list, save): +def _box_whisker(pkl_list, name_list, save, cut=sys.maxint): plotting_dir = os.path.dirname(os.path.realpath(__file__)) cur_dir = os.getcwd() @@ -109,7 +111,7 @@ def _box_whisker(pkl_list, name_list, save): try: os.chdir(plotting_dir) import plotBoxWhisker - plotBoxWhisker.main(pkl_list, name_list, save=save) + plotBoxWhisker.main(pkl_list, name_list, save=save, cut=cut) os.chdir(cur_dir) sys.stdout.write("passed\n") except Exception, e: @@ -117,14 +119,14 @@ def _box_whisker(pkl_list, name_list, save): sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) -def _generate_tex_table(pkl_list, name_list, save): +def _generate_tex_table(pkl_list, name_list, save, cut=sys.maxint): plotting_dir = os.path.dirname(os.path.realpath(__file__)) cur_dir = os.getcwd() # noinspection PyBroadException try: os.chdir(plotting_dir) import generateTexTable - table = generateTexTable.main(pkl_list, name_list) + table = generateTexTable.main(pkl_list, name_list, cut) with open(save, "w") as fh: fh.write(table) os.chdir(cur_dir) @@ -134,14 +136,14 @@ def _generate_tex_table(pkl_list, name_list, save): sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) -def _statistics(pkl_list, name_list, save=""): +def _statistics(pkl_list, name_list, save="", cut=sys.maxint): plotting_dir = os.path.dirname(os.path.realpath(__file__)) cur_dir = os.getcwd() # noinspection PyBroadException try: os.chdir(plotting_dir) - cmd = ["python statistics.py"] + cmd = ["python statistics.py", "--cut %d" % cut] for i in range(len(name_list)): cmd.append(name_list[i][0]) for pkl in pkl_list[i]: @@ -174,9 +176,10 @@ def main(): default=False, help="Do NOT plot on log scale") parser.add_argument("-s", "--save", dest="save", default="", help="Where to save plots? (directory)") - parser.add_argument("-f", "--file", dest="file", default="png", help="File ending") + parser.add_argument("-c", "--cut", dest="cut", default=sys.maxint, + type=int, help="Cut experiment pickles after a specified number of trials.") args, unknown = parser.parse_known_args() @@ -210,7 +213,8 @@ def main(): else: tmp_save = save_dir sys.stdout.write("plotTrace.py ... %s ..." % tmp_save) - _plot_trace(pkl_list=pkl_list, name_list=name_list, save=tmp_save, log=log) + _plot_trace(pkl_list=pkl_list, name_list=name_list, save=tmp_save, + log=log, cut=args.cut) if len(name_list) > 1: # Some plots only make sense, if there are many experiments @@ -220,7 +224,8 @@ def main(): else: tmp_save = save_dir sys.stdout.write("plotBoxWhisker.py ... %s ..." % tmp_save) - _box_whisker(pkl_list=pkl_list, name_list=name_list, save=tmp_save) + _box_whisker(pkl_list=pkl_list, name_list=name_list, save=tmp_save, + cut=args.cut) # statistics if save_dir is not "": @@ -228,7 +233,8 @@ def main(): else: tmp_save = save_dir sys.stdout.write("statistics.py ... %s ..." % tmp_save) - _statistics(pkl_list=pkl_list, name_list=name_list, save=tmp_save) + _statistics(pkl_list=pkl_list, name_list=name_list, save=tmp_save, + cut=args.cut) # LaTeX table if save_dir is not "": @@ -236,7 +242,7 @@ def main(): else: tmp_save = save_dir sys.stdout.write("generateTexTable.py ... %s ..." % tmp_save) - _generate_tex_table(pkl_list, name_list, tmp_save) + _generate_tex_table(pkl_list, name_list, tmp_save, cut=args.cut) # We can always plot this @@ -246,7 +252,8 @@ def main(): else: tmp_save = save_dir sys.stdout.write("plotOptimizerOverhead.py ... %s ..." % tmp_save) - _optimizer_overhead(pkl_list=pkl_list, name_list=name_list, save=tmp_save, log=log) + _optimizer_overhead(pkl_list=pkl_list, name_list=name_list, + save=tmp_save, log=log, cut=args.cut) # Error Trace with Std if save_dir is not "": @@ -254,14 +261,16 @@ def main(): else: tmp_save = save_dir sys.stdout.write("TraceWithStd_perEval.py ... %s ..." % tmp_save) - _trace_with_std_per_eval(pkl_list=pkl_list, name_list=name_list, save=tmp_save, log=log) + _trace_with_std_per_eval(pkl_list=pkl_list, name_list=name_list, + save=tmp_save, log=log, cut=args.cut) if save_dir is not "": tmp_save = os.path.join(save_dir, "TraceWithStd_perTime_%s.%s" % (time_str, args.file)) else: tmp_save = save_dir sys.stdout.write("TraceWithStd_perTime.py ... %s ..." % tmp_save) - _trace_with_std_per_time(pkl_list=pkl_list, name_list=name_list, save=tmp_save, log=log) + _trace_with_std_per_time(pkl_list=pkl_list, name_list=name_list, + save=tmp_save, log=log, cut=args.cut) if __name__ == "__main__": main() \ No newline at end of file diff --git a/HPOlib/Plotting/generateTexTable.py b/HPOlib/Plotting/generateTexTable.py index 7348530f..4e2bf201 100644 --- a/HPOlib/Plotting/generateTexTable.py +++ b/HPOlib/Plotting/generateTexTable.py @@ -124,7 +124,7 @@ def main(pkl_list, name_list, cut=sys.maxint): parser = ArgumentParser(description=description, prog=prog) - parser.add_argument("-c", "--c", dest="cut", default=sys.maxint, + parser.add_argument("-c", "--cut", dest="cut", default=sys.maxint, type=int, help="Only consider that many evaluations") args, unknown = parser.parse_known_args() # TODO-list: diff --git a/HPOlib/Plotting/plotBoxWhisker.py b/HPOlib/Plotting/plotBoxWhisker.py index b5882187..aaa75f85 100644 --- a/HPOlib/Plotting/plotBoxWhisker.py +++ b/HPOlib/Plotting/plotBoxWhisker.py @@ -35,7 +35,8 @@ __contact__ = "automl.org" -def plot_box_whisker(best_trials, name_list, title="", save="", y_min=0, y_max=0): +def plot_box_whisker(best_trials, name_list, title="", save="", y_min=0, + y_max=0): ratio = 5 gs = GridSpec(ratio, 1) fig = figure(1, dpi=100) @@ -104,7 +105,7 @@ def plot_box_whisker(best_trials, name_list, title="", save="", y_min=0, y_max=0 show() -def main(pkl_list, name_list, title="", save="", y_min=0, y_max=0): +def main(pkl_list, name_list, title="", save="", y_min=0, y_max=0, cut=sys.maxint): best_trials = list() for i in range(len(name_list)): @@ -113,7 +114,7 @@ def main(pkl_list, name_list, title="", save="", y_min=0, y_max=0): fh = open(pkl, "r") trials = cPickle.load(fh) fh.close() - best_trials[i].append(plot_util.get_best(trials)) + best_trials[i].append(plot_util.get_best(trials, cut=cut)) plot_box_whisker(best_trials=best_trials, name_list=name_list, title=title, save=save, y_min=y_min, y_max=y_max) @@ -138,6 +139,8 @@ def main(pkl_list, name_list, title="", save="", y_min=0, y_max=0): type=float, help="Minimum of the plot") parser.add_argument("-s", "--save", dest="save", default="", help="Where to save plot instead of showing it?") + parser.add_argument("-c", "--cut", default=sys.maxint, type=int, + help="Cut experiment pickles after a specified number of trials.") args, unknown = parser.parse_known_args() sys.stdout.write("\nFound " + str(len(unknown)) + " arguments...") @@ -145,4 +148,4 @@ def main(pkl_list, name_list, title="", save="", y_min=0, y_max=0): pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) main(pkl_list=pkl_list_main, name_list=name_list_main, title=args.title, save=args.save, - y_min=args.min, y_max=args.max) + y_min=args.min, y_max=args.max, cut=args.cut) diff --git a/HPOlib/Plotting/plotOptimizerOverhead.py b/HPOlib/Plotting/plotOptimizerOverhead.py index d7f905d4..bfbeedfb 100644 --- a/HPOlib/Plotting/plotOptimizerOverhead.py +++ b/HPOlib/Plotting/plotOptimizerOverhead.py @@ -106,7 +106,9 @@ def plot_time_trace(time_dict, name_list, title="", log=True, save="", y_max=0, plt.show() -def main(pkl_list, name_list, autofill, title="", log=False, save="", y_min=0, y_max=0): +def main(pkl_list, name_list, autofill, title="", log=False, save="", + y_min=0, y_max=0, cut=sys.maxint): + times_dict = dict() for exp in range(len(name_list)): times_dict[name_list[exp][0]] = list() @@ -115,34 +117,38 @@ def main(pkl_list, name_list, autofill, title="", log=False, save="", y_min=0, y trials = cPickle.load(fh) fh.close() + # Get all variables from the trials object + cv_starttime = trials["cv_starttime"][:cut] + cv_endtime = trials["cv_endtime"][:cut] + # Get optimizer duration times time_list = list() # First duration - time_list.append(trials["cv_starttime"][0] - trials["starttime"][0]) + time_list.append(cv_starttime[0] - trials["starttime"][0]) time_idx = 0 - for i in range(len(trials["cv_starttime"][1:])): + for i in range(len(cv_starttime[1:])): # Is there a next restored run? # if yes, does next cvstart belong to a restored run? if len(trials["endtime"]) > time_idx and \ - trials["cv_starttime"][i+1] > trials["endtime"][time_idx]: + cv_starttime[i+1] > trials["endtime"][time_idx]: # Check whether run crashed/terminated during cv # Equals means that the run crashed - if trials["cv_endtime"][i] < trials["endtime"][time_idx]: + if cv_endtime[i] < trials["endtime"][time_idx]: # No .. everything is fine - time_list.append((trials["endtime"][time_idx] - trials["cv_endtime"][i])) - time_list.append((trials["cv_starttime"][i + 1] - trials["starttime"][time_idx+1])) - elif trials["endtime"][time_idx] == trials["cv_endtime"][i]: + time_list.append((trials["endtime"][time_idx] - cv_endtime[i])) + time_list.append((cv_starttime[i + 1] - trials["starttime"][time_idx+1])) + elif trials["endtime"][time_idx] == cv_endtime[i]: # Yes, but BBoM managed to set an endtime pass else: # Yes ... trouble print "Help" print trials["endtime"][time_idx] - print trials["cv_endtime"][i] + print cv_endtime[i] time_idx += 1 # everything ... else: - time_list.append(trials["cv_starttime"][i + 1] - trials["cv_endtime"][i]) + time_list.append(cv_starttime[i + 1] - cv_endtime[i]) times_dict[name_list[exp][0]].append(time_list) for key in times_dict.keys(): @@ -171,6 +177,8 @@ def main(pkl_list, name_list, autofill, title="", log=False, save="", y_min=0, y parser = ArgumentParser(description=description, prog=prog) # General Options + parser.add_argument("-c", "--cut", type=int, default=sys.maxint, + help="Cut the experiment pickle length.") parser.add_argument("-l", "--nolog", action="store_true", dest="log", default=False, help="Do NOT plot on log scale") parser.add_argument("--max", type=float, dest="max", @@ -191,5 +199,6 @@ def main(pkl_list, name_list, autofill, title="", log=False, save="", y_min=0, y sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n") pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) - main(pkl_list_main, name_list_main, autofill=args.autofill, title=args.title, log=not args.log, save=args.save, - y_min=args.min, y_max=args.max) + main(pkl_list_main, name_list_main, autofill=args.autofill, title=args.title, + log=not args.log, save=args.save, y_min=args.min, y_max=args.max, + cut=args.cut) diff --git a/HPOlib/Plotting/plotTrace.py b/HPOlib/Plotting/plotTrace.py index 94fb2ee9..bc22f3e8 100644 --- a/HPOlib/Plotting/plotTrace.py +++ b/HPOlib/Plotting/plotTrace.py @@ -35,7 +35,7 @@ def plot_optimization_trace(trial_list, name_list, optimum=0, title="", log=False, - save="", y_min=0, y_max=0): + save="", y_min=0, y_max=0, cut=sys.maxint): markers = itertools.cycle(['o', 's', '^', 'x']) colors = itertools.cycle(['b', 'g', 'r', 'k']) linestyles = itertools.cycle(['-']) @@ -55,10 +55,13 @@ def plot_optimization_trace(trial_list, name_list, optimum=0, title="", log=Fals # This might not do what we actually want; Ideally it would only take the # ones that where actually run according to instance_order for i in range(len(name_list)): - x = range(len(trial_list[i])) - y = np.zeros((len(trial_list[i]))) - line = np.zeros((len(trial_list[i]))) - for j, inst_res in enumerate(trial_list[i]): + print cut, len(trial_list[i]) + num_plotted_trials = np.min([cut, len(trial_list[i])]) + print num_plotted_trials + x = range(num_plotted_trials) + y = np.zeros((num_plotted_trials)) + line = np.zeros((num_plotted_trials)) + for j, inst_res in enumerate(trial_list[i][:num_plotted_trials]): if j >= len(y): break if type(inst_res) == np.ndarray and not np.isfinite(inst_res).any(): @@ -86,8 +89,8 @@ def plot_optimization_trace(trial_list, name_list, optimum=0, title="", log=Fals min_val = min(y) if max(y) > max_val: max_val = max(y) - if len(trial_list[i]) > max_trials: - max_trials = len(trial_list[i]) + if num_plotted_trials > max_trials: + max_trials = num_plotted_trials # Describe and label the stuff ax.set_xlabel("#Function evaluations") @@ -116,7 +119,8 @@ def plot_optimization_trace(trial_list, name_list, optimum=0, title="", log=Fals show() -def main(pkl_list, name_list, optimum=0, title="", log=False, save="", y_max=0, y_min=0): +def main(pkl_list, name_list, optimum=0, title="", log=False, save="", y_max=0, + y_min=0, cut=sys.maxint): trial_list = list() for i in range(len(pkl_list)): @@ -129,8 +133,8 @@ def main(pkl_list, name_list, optimum=0, title="", log=False, save="", y_max=0, sys.stdout.write("Plotting trace\n") plot_optimization_trace(trial_list=trial_list, name_list=name_list, optimum=optimum, - title=title, log=not log, - save=save, y_max=y_max, y_min=y_min) + title=title, log=not log, save=save, y_max=y_max, + y_min=y_min, cut=cut) if save != "": sys.stdout.write("Saved plot to " + save + "\n") @@ -157,6 +161,8 @@ def main(pkl_list, name_list, optimum=0, title="", log=False, save="", y_max=0, default="", help="Where to save plot instead of showing it?") parser.add_argument("-t", "--title", dest="title", default="", help="Optional supertitle for plot") + parser.add_argument("-c", "--cut", default=sys.maxint, type=int, + help="Cut experiment pickles after a specified number of trials.") args, unknown = parser.parse_known_args() @@ -165,4 +171,5 @@ def main(pkl_list, name_list, optimum=0, title="", log=False, save="", y_max=0, pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) main(pkl_list=pkl_list_main, name_list=name_list_main, optimum=args.optimum, - title=args.title, log=args.log, save=args.save, y_max=args.max, y_min=args.min) + title=args.title, log=args.log, save=args.save, y_max=args.max, + y_min=args.min, cut=args.cut) diff --git a/HPOlib/Plotting/plotTraceWithStd_perEval.py b/HPOlib/Plotting/plotTraceWithStd_perEval.py index 41b3d15f..687e8768 100644 --- a/HPOlib/Plotting/plotTraceWithStd_perEval.py +++ b/HPOlib/Plotting/plotTraceWithStd_perEval.py @@ -117,7 +117,8 @@ def plot_optimization_trace(trial_list, name_list, optimum=0, title="", show() -def main(pkl_list, name_list, autofill, optimum=0, save="", title="", log=False, y_min=0, y_max=0, scale_std=1): +def main(pkl_list, name_list, autofill, optimum=0, save="", title="", log=False, + y_min=0, y_max=0, scale_std=1, cut=sys.maxint): trial_list = list() for i in range(len(pkl_list)): @@ -127,7 +128,7 @@ def main(pkl_list, name_list, autofill, optimum=0, save="", title="", log=False, trials = cPickle.load(fh) fh.close() - trace = plot_util.extract_trajectory(trials) + trace = plot_util.extract_trajectory(trials, cut=cut) trial_list[-1].append(np.array(trace)) for i in range(len(trial_list)): diff --git a/HPOlib/Plotting/plotTraceWithStd_perTime.py b/HPOlib/Plotting/plotTraceWithStd_perTime.py index c91b9266..6578e0f2 100644 --- a/HPOlib/Plotting/plotTraceWithStd_perTime.py +++ b/HPOlib/Plotting/plotTraceWithStd_perTime.py @@ -183,7 +183,8 @@ def fill_trajectories(trace_list, times_list): return trajectories, times -def main(pkl_list, name_list, autofill, optimum=0, save="", title="", log=False, y_min=0, y_max=0, scale_std=1): +def main(pkl_list, name_list, autofill, optimum=0, save="", title="", + log=False, y_min=0, y_max=0, scale_std=1, cut=sys.maxint): trial_list = list() times_list = list() diff --git a/HPOlib/Plotting/plot_util.py b/HPOlib/Plotting/plot_util.py index 47ce9e99..fc3ecce0 100644 --- a/HPOlib/Plotting/plot_util.py +++ b/HPOlib/Plotting/plot_util.py @@ -68,27 +68,27 @@ def get_pkl_and_name_list(argument_list): return pkl_list, name_list -def extract_trajectory(trials): +def extract_trajectory(trials, cut=sys.maxint): trace = list() - currentbest = np.nanmax(np.array([trial["result"] for trial in trials['trials']])) + currentbest = trials['trials'][0] - for result in [trial["result"] for trial in trials['trials']]: + for result in [trial["result"] for trial in trials['trials'][:cut]]: if result < currentbest: currentbest = result trace.append(currentbest) return trace -def extract_trials(trials): - trl = [trial["result"] for trial in trials['trials']] +def extract_trials(trials, cut=sys.maxint): + trl = [trial["result"] for trial in trials['trials'][:cut]] return trl -def extract_runtime_timestamps(trials): +def extract_runtime_timestamps(trials, cut=sys.maxint): # return a list like (20, 53, 101, 200) time_list = list() time_list.append(0) - for trial in trials["trials"]: + for trial in trials["trials"][:cut]: time_list.append(np.sum(trial["instance_durations"]) + time_list[-1]) return time_list @@ -111,7 +111,7 @@ def get_best_value_and_index(trials, cut=False): best_index = -1 if cut and 0 < cut < len(traj): best_value = traj[cut] - best_index = np.argmin(traj[cut]) + best_index = np.argmin(traj[:cut]) else: best_value = traj[-1] best_index = np.argmin(traj) diff --git a/HPOlib/Plotting/statistics.py b/HPOlib/Plotting/statistics.py index 8ad01563..62b14100 100644 --- a/HPOlib/Plotting/statistics.py +++ b/HPOlib/Plotting/statistics.py @@ -112,8 +112,9 @@ def _mann_whitney_u(x, y=None): """ -def main(pkl_list, name_list): - best_dict, idx_dict, keys = plot_util.read_pickles(name_list, pkl_list) +def main(pkl_list, name_list, cut=sys.maxint): + best_dict, idx_dict, keys = plot_util.read_pickles(name_list, pkl_list, + cut=cut) for k in keys: sys.stdout.write("%10s: %s experiment(s)\n" % (k, len(best_dict[k]))) @@ -169,9 +170,9 @@ def main(pkl_list, name_list): parser = ArgumentParser(description=description, prog=prog) - parser.add_argument("-c", "--c", dest="cut", default=False, + parser.add_argument("-c", "--cut", dest="cut", default=sys.maxint, type=int, help="Only consider that many evaluations") args, unknown = parser.parse_known_args() pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) - main(pkl_list=pkl_list_main, name_list=name_list_main) + main(pkl_list=pkl_list_main, name_list=name_list_main, cut=args.cut) From fb55cecdc83961eee1c4d88bb5c9588b13774e93 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 24 Mar 2014 16:55:57 +0100 Subject: [PATCH 127/139] FIX: benchmark_functions.lda_on_grid returns time and perplexity also rename lda_ongrid to lda_on_grid --- HPOlib/benchmark_functions.py | 2 +- benchmarks/lda_on_grid/config.cfg | 13 ++++++ .../hyperopt_august2013_mod/space.py | 5 +++ benchmarks/lda_on_grid/lda.py | 40 +++++++++++++++++++ .../random_hyperopt_august2013_mod | 1 + .../lda_on_grid/smac_2_06_01-dev/params.pcs | 4 ++ .../spearmint_april2013_mod/config.pb | 27 +++++++++++++ 7 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 benchmarks/lda_on_grid/config.cfg create mode 100644 benchmarks/lda_on_grid/hyperopt_august2013_mod/space.py create mode 100644 benchmarks/lda_on_grid/lda.py create mode 120000 benchmarks/lda_on_grid/random_hyperopt_august2013_mod create mode 100644 benchmarks/lda_on_grid/smac_2_06_01-dev/params.pcs create mode 100644 benchmarks/lda_on_grid/spearmint_april2013_mod/config.pb diff --git a/HPOlib/benchmark_functions.py b/HPOlib/benchmark_functions.py index c352e984..5a2eb531 100644 --- a/HPOlib/benchmark_functions.py +++ b/HPOlib/benchmark_functions.py @@ -442,7 +442,7 @@ def lda_on_grid(kappa, tau, s, ret_time=False): if ret_time: return config_tree[kappa_values[kappa]][tau_values[tau]][s_values[s]][1] else: - return config_tree[kappa_values[kappa]][tau_values[tau]][s_values[s]][1] + return config_tree[kappa_values[kappa]][tau_values[tau]][s_values[s]][0] def save_svm_on_grid(params, ret_time=False, **kwargs): diff --git a/benchmarks/lda_on_grid/config.cfg b/benchmarks/lda_on_grid/config.cfg new file mode 100644 index 00000000..6c245bdb --- /dev/null +++ b/benchmarks/lda_on_grid/config.cfg @@ -0,0 +1,13 @@ +[SMAC] +p = smac_2_06_01-dev/params.pcs + +[TPE] +space = space.py + +[SPEARMINT] +config = config.pb + +[HPOLIB] +function = python ../lda.py +number_of_jobs = 50 +result_on_terminate = 1000000000 diff --git a/benchmarks/lda_on_grid/hyperopt_august2013_mod/space.py b/benchmarks/lda_on_grid/hyperopt_august2013_mod/space.py new file mode 100644 index 00000000..3b8ead30 --- /dev/null +++ b/benchmarks/lda_on_grid/hyperopt_august2013_mod/space.py @@ -0,0 +1,5 @@ +from hyperopt import hp + +space = {'Kappa': hp.quniform('Kappa', 0, 5, 1), + 'Tau': hp.quniform('Tau', 0, 5, 1), + 'S': hp.quniform('S', 0, 7, 1)} diff --git a/benchmarks/lda_on_grid/lda.py b/benchmarks/lda_on_grid/lda.py new file mode 100644 index 00000000..c21a1125 --- /dev/null +++ b/benchmarks/lda_on_grid/lda.py @@ -0,0 +1,40 @@ +## +# wrapping: A program making it easy to use hyperparameter +# optimization software. +# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import HPOlib.benchmark_util as benchmark_util +import HPOlib.benchmark_functions as benchmark_functions + +__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] +__contact__ = "automl.org" +__credits__ = ["Jasper Snoek", "Ryan P. Adams", "Hugo Larochelle"] + + +def main(params, ret_time=False, **kwargs): + print 'Params: ', params + y = benchmark_functions.save_lda_on_grid(params, ret_time=ret_time, **kwargs) + print 'Result: ', y + return y + + +if __name__ == "__main__": + args, params = benchmark_util.parse_cli() + result = main(params, ret_time=False, **args) + duration = main(params, ret_time=True) + print "Result for ParamILS: %s, %f, 1, %f, %d, %s" % \ + ("SAT", abs(duration), result, -1, str(__file__)) \ No newline at end of file diff --git a/benchmarks/lda_on_grid/random_hyperopt_august2013_mod b/benchmarks/lda_on_grid/random_hyperopt_august2013_mod new file mode 120000 index 00000000..c4f9ce32 --- /dev/null +++ b/benchmarks/lda_on_grid/random_hyperopt_august2013_mod @@ -0,0 +1 @@ +hyperopt_august2013_mod \ No newline at end of file diff --git a/benchmarks/lda_on_grid/smac_2_06_01-dev/params.pcs b/benchmarks/lda_on_grid/smac_2_06_01-dev/params.pcs new file mode 100644 index 00000000..c3551e94 --- /dev/null +++ b/benchmarks/lda_on_grid/smac_2_06_01-dev/params.pcs @@ -0,0 +1,4 @@ +Kappa [0, 5] [0]i +Tau [0, 5] [0]i +S [0, 7] [0]i + diff --git a/benchmarks/lda_on_grid/spearmint_april2013_mod/config.pb b/benchmarks/lda_on_grid/spearmint_april2013_mod/config.pb new file mode 100644 index 00000000..0dff3cdb --- /dev/null +++ b/benchmarks/lda_on_grid/spearmint_april2013_mod/config.pb @@ -0,0 +1,27 @@ +language: PYTHON +name: "HPOlib.cv" + +variable { + name: "Kappa" + type: INT + size: 1 + min: 0 + max: 5 +} + +variable { + name: "Tau" + type: INT + size: 1 + min: 0 + max: 5 +} + +variable { + name: "S" + type: INT + size: 1 + min: 0 + max: 7 +} + From 7c1b76dd0960b865e678ceab6336d03e5024d462 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Mon, 24 Mar 2014 17:10:07 +0100 Subject: [PATCH 128/139] REFACTOR: flipped behaviour of boolean log flag in the plotting files --- HPOlib/Plotting/doAllPlots.py | 33 +++++++++++---------- HPOlib/Plotting/plotOptimizerOverhead.py | 8 ++--- HPOlib/Plotting/plotTrace.py | 6 ++-- HPOlib/Plotting/plotTraceWithStd_perEval.py | 6 ++-- HPOlib/Plotting/plotTraceWithStd_perTime.py | 6 ++-- 5 files changed, 31 insertions(+), 28 deletions(-) diff --git a/HPOlib/Plotting/doAllPlots.py b/HPOlib/Plotting/doAllPlots.py index deb33406..b196577a 100644 --- a/HPOlib/Plotting/doAllPlots.py +++ b/HPOlib/Plotting/doAllPlots.py @@ -50,8 +50,8 @@ def _plot_trace(pkl_list, name_list, save="", cut=sys.maxint, log=False): sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) -def _trace_with_std_per_eval(pkl_list, name_list, save="", log=False, - cut=sys.maxint): +def _trace_with_std_per_eval(pkl_list, name_list, save="", + cut=sys.maxint, log=False): plotting_dir = os.path.dirname(os.path.realpath(__file__)) cur_dir = os.getcwd() @@ -68,8 +68,8 @@ def _trace_with_std_per_eval(pkl_list, name_list, save="", log=False, sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) -def _trace_with_std_per_time(pkl_list, name_list, save="", log=False, - cut=sys.maxint): +def _trace_with_std_per_time(pkl_list, name_list, save="", + cut=sys.maxint, log=False): plotting_dir = os.path.dirname(os.path.realpath(__file__)) cur_dir = os.getcwd() @@ -86,7 +86,8 @@ def _trace_with_std_per_time(pkl_list, name_list, save="", log=False, sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) -def _optimizer_overhead(pkl_list, name_list, save, log, cut=sys.maxint): +def _optimizer_overhead(pkl_list, name_list, save="", + cut=sys.maxint, log=False): plotting_dir = os.path.dirname(os.path.realpath(__file__)) cur_dir = os.getcwd() @@ -103,7 +104,7 @@ def _optimizer_overhead(pkl_list, name_list, save, log, cut=sys.maxint): sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) -def _box_whisker(pkl_list, name_list, save, cut=sys.maxint): +def _box_whisker(pkl_list, name_list, save="", cut=sys.maxint, log=False): plotting_dir = os.path.dirname(os.path.realpath(__file__)) cur_dir = os.getcwd() @@ -119,7 +120,8 @@ def _box_whisker(pkl_list, name_list, save, cut=sys.maxint): sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) -def _generate_tex_table(pkl_list, name_list, save, cut=sys.maxint): +def _generate_tex_table(pkl_list, name_list, save="", cut=sys.maxint, + log=False): plotting_dir = os.path.dirname(os.path.realpath(__file__)) cur_dir = os.getcwd() # noinspection PyBroadException @@ -136,7 +138,7 @@ def _generate_tex_table(pkl_list, name_list, save, cut=sys.maxint): sys.stderr.write("failed: %s %s" % (sys.exc_info()[0], e)) -def _statistics(pkl_list, name_list, save="", cut=sys.maxint): +def _statistics(pkl_list, name_list, save="", cut=sys.maxint, log=False): plotting_dir = os.path.dirname(os.path.realpath(__file__)) cur_dir = os.getcwd() @@ -172,8 +174,8 @@ def main(): parser = ArgumentParser(description=description, prog=prog) # General Options - parser.add_argument("-l", "--nolog", action="store_true", dest="log", - default=False, help="Do NOT plot on log scale") + parser.add_argument("-l", "--log", action="store_true", dest="log", + default=False, help="Plot on log scale") parser.add_argument("-s", "--save", dest="save", default="", help="Where to save plots? (directory)") parser.add_argument("-f", "--file", dest="file", @@ -225,7 +227,7 @@ def main(): tmp_save = save_dir sys.stdout.write("plotBoxWhisker.py ... %s ..." % tmp_save) _box_whisker(pkl_list=pkl_list, name_list=name_list, save=tmp_save, - cut=args.cut) + log=log, cut=args.cut) # statistics if save_dir is not "": @@ -234,7 +236,7 @@ def main(): tmp_save = save_dir sys.stdout.write("statistics.py ... %s ..." % tmp_save) _statistics(pkl_list=pkl_list, name_list=name_list, save=tmp_save, - cut=args.cut) + log=log, cut=args.cut) # LaTeX table if save_dir is not "": @@ -242,7 +244,8 @@ def main(): else: tmp_save = save_dir sys.stdout.write("generateTexTable.py ... %s ..." % tmp_save) - _generate_tex_table(pkl_list, name_list, tmp_save, cut=args.cut) + _generate_tex_table(pkl_list=pkl_list, name_list=name_list, + save=tmp_save, log=log, cut=args.cut) # We can always plot this @@ -252,8 +255,8 @@ def main(): else: tmp_save = save_dir sys.stdout.write("plotOptimizerOverhead.py ... %s ..." % tmp_save) - _optimizer_overhead(pkl_list=pkl_list, name_list=name_list, - save=tmp_save, log=log, cut=args.cut) + _optimizer_overhead(pkl_list=pkl_list, name_list=name_list, save=tmp_save, + log=log, cut=args.cut) # Error Trace with Std if save_dir is not "": diff --git a/HPOlib/Plotting/plotOptimizerOverhead.py b/HPOlib/Plotting/plotOptimizerOverhead.py index bfbeedfb..3cf65051 100644 --- a/HPOlib/Plotting/plotOptimizerOverhead.py +++ b/HPOlib/Plotting/plotOptimizerOverhead.py @@ -162,7 +162,7 @@ def main(pkl_list, name_list, autofill, title="", log=False, save="", raise ValueError("(%s != %s), Traces do not have the same length, please use -a" % (str(max_len), str(len(times_dict[key][t])))) - plot_time_trace(times_dict, name_list, title=title, log=not log, save=save, + plot_time_trace(times_dict, name_list, title=title, log=log, save=save, y_min=y_min, y_max=y_max) if save != "": @@ -179,8 +179,8 @@ def main(pkl_list, name_list, autofill, title="", log=False, save="", # General Options parser.add_argument("-c", "--cut", type=int, default=sys.maxint, help="Cut the experiment pickle length.") - parser.add_argument("-l", "--nolog", action="store_true", dest="log", - default=False, help="Do NOT plot on log scale") + parser.add_argument("-l", "--log", action="store_true", dest="log", + default=False, help="Plot on log scale") parser.add_argument("--max", type=float, dest="max", default=0, help="Maximum of the plot") parser.add_argument("--min", type=float, dest="min", @@ -200,5 +200,5 @@ def main(pkl_list, name_list, autofill, title="", log=False, save="", pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) main(pkl_list_main, name_list_main, autofill=args.autofill, title=args.title, - log=not args.log, save=args.save, y_min=args.min, y_max=args.max, + log=args.log, save=args.save, y_min=args.min, y_max=args.max, cut=args.cut) diff --git a/HPOlib/Plotting/plotTrace.py b/HPOlib/Plotting/plotTrace.py index bc22f3e8..7c220431 100644 --- a/HPOlib/Plotting/plotTrace.py +++ b/HPOlib/Plotting/plotTrace.py @@ -133,7 +133,7 @@ def main(pkl_list, name_list, optimum=0, title="", log=False, save="", y_max=0, sys.stdout.write("Plotting trace\n") plot_optimization_trace(trial_list=trial_list, name_list=name_list, optimum=optimum, - title=title, log=not log, save=save, y_max=y_max, + title=title, log=log, save=save, y_max=y_max, y_min=y_min, cut=cut) if save != "": @@ -151,8 +151,8 @@ def main(pkl_list, name_list, optimum=0, title="", log=False, save="", y_max=0, parser.add_argument("-o", "--optimum", type=float, dest="optimum", default=0, help="If not set, the optimum is supposed to be zero") # General Options - parser.add_argument("-l", "--nolog", action="store_true", dest="log", - default=False, help="Do NOT plot on log scale") + parser.add_argument("-l", "--log", action="store_true", dest="log", + default=False, help="Plot on log scale") parser.add_argument("--max", dest="max", type=float, default=0, help="Maximum of the plot") parser.add_argument("--min", dest="min", type=float, diff --git a/HPOlib/Plotting/plotTraceWithStd_perEval.py b/HPOlib/Plotting/plotTraceWithStd_perEval.py index 687e8768..64d1fd43 100644 --- a/HPOlib/Plotting/plotTraceWithStd_perEval.py +++ b/HPOlib/Plotting/plotTraceWithStd_perEval.py @@ -142,7 +142,7 @@ def main(pkl_list, name_list, autofill, optimum=0, save="", title="", log=False, raise ValueError("(%s != %s), Traces do not have the same length, please use -a" % (str(max_len), str(len(trial_list[i][t])))) - plot_optimization_trace(trial_list, name_list, optimum, title=title, log=not log, + plot_optimization_trace(trial_list, name_list, optimum, title=title, log=log, save=save, y_min=y_min, y_max=y_max, scale_std=scale_std) if save != "": @@ -166,8 +166,8 @@ def main(pkl_list, name_list, autofill, optimum=0, save="", title="", log=False, parser.add_argument("-c", "--scale", type=float, dest="scale", default=1, help="Multiply std to get a nicer plot") # General Options - parser.add_argument("-l", "--nolog", action="store_true", dest="log", - default=False, help="Do NOT plot on log scale") + parser.add_argument("-l", "--log", action="store_true", dest="log", + default=False, help="Plot on log scale") parser.add_argument("--max", dest="max", type=float, default=0, help="Maximum of the plot") parser.add_argument("--min", dest="min", type=float, diff --git a/HPOlib/Plotting/plotTraceWithStd_perTime.py b/HPOlib/Plotting/plotTraceWithStd_perTime.py index 6578e0f2..432d77e2 100644 --- a/HPOlib/Plotting/plotTraceWithStd_perTime.py +++ b/HPOlib/Plotting/plotTraceWithStd_perTime.py @@ -217,7 +217,7 @@ def main(pkl_list, name_list, autofill, optimum=0, save="", title="", raise ValueError("(%s != %s), Traces do not have the same length, please use -a" % (str(max_len), str(len(trial_list[i][t])))) - plot_optimization_trace(trial_list, name_list, times_list, optimum, title=title, log=not log, + plot_optimization_trace(trial_list, name_list, times_list, optimum, title=title, log=log, save=save, y_min=y_min, y_max=y_max, scale_std=scale_std) if save != "": sys.stdout.write("Saved plot to " + save + "\n") @@ -240,8 +240,8 @@ def main(pkl_list, name_list, autofill, optimum=0, save="", title="", parser.add_argument("-c", "--scale", type=float, dest="scale", default=1, help="Multiply std to get a nicer plot") # General Options - parser.add_argument("-l", "--nolog", action="store_true", dest="log", - default=False, help="Do NOT plot on log scale") + parser.add_argument("-l", "--log", action="store_true", dest="log", + default=False, help="Plot on log scale") parser.add_argument("--max", dest="max", type=float, default=0, help="Maximum of the plot") parser.add_argument("--min", dest="min", type=float, From de4bf2d563625c0202bdf924311f01509cac63cd Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Mon, 24 Mar 2014 18:04:29 +0100 Subject: [PATCH 129/139] REFACTOR: changed default colors of plots --- HPOlib/Plotting/plotOptimizerOverhead.py | 4 ++-- HPOlib/Plotting/plotTrace.py | 4 ++-- HPOlib/Plotting/plotTraceWithStd_perEval.py | 4 ++-- HPOlib/Plotting/plotTraceWithStd_perTime.py | 9 ++------- HPOlib/Plotting/plot_util.py | 18 ++++++++++++++++++ 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/HPOlib/Plotting/plotOptimizerOverhead.py b/HPOlib/Plotting/plotOptimizerOverhead.py index 3cf65051..35c3538f 100644 --- a/HPOlib/Plotting/plotOptimizerOverhead.py +++ b/HPOlib/Plotting/plotOptimizerOverhead.py @@ -34,8 +34,8 @@ def plot_time_trace(time_dict, name_list, title="", log=True, save="", y_max=0, y_min=0): - colors = itertools.cycle(['b', 'g', 'r', 'k']) - markers = itertools.cycle(['o', 's', 'x', '^']) + colors = plot_util.get_plot_colors() + markers = plot_util.get_plot_markers() linestyles = itertools.cycle(['-']) size = 5 diff --git a/HPOlib/Plotting/plotTrace.py b/HPOlib/Plotting/plotTrace.py index 7c220431..b5cf3b9d 100644 --- a/HPOlib/Plotting/plotTrace.py +++ b/HPOlib/Plotting/plotTrace.py @@ -36,8 +36,8 @@ def plot_optimization_trace(trial_list, name_list, optimum=0, title="", log=False, save="", y_min=0, y_max=0, cut=sys.maxint): - markers = itertools.cycle(['o', 's', '^', 'x']) - colors = itertools.cycle(['b', 'g', 'r', 'k']) + markers = plot_util.get_plot_markers() + colors = plot_util.get_plot_colors() linestyles = itertools.cycle(['-']) size = 1 diff --git a/HPOlib/Plotting/plotTraceWithStd_perEval.py b/HPOlib/Plotting/plotTraceWithStd_perEval.py index 64d1fd43..a7230061 100644 --- a/HPOlib/Plotting/plotTraceWithStd_perEval.py +++ b/HPOlib/Plotting/plotTraceWithStd_perEval.py @@ -35,8 +35,8 @@ def plot_optimization_trace(trial_list, name_list, optimum=0, title="", log=True, save="", y_max=0, y_min=0, scale_std=1): - markers = itertools.cycle(['o', 's', 'x', '^']) - colors = itertools.cycle(['b', 'g', 'r', 'k']) + markers =plot_util.get_plot_markers() + colors = plot_util.get_plot_colors() linestyles = itertools.cycle(['-']) size = 1 diff --git a/HPOlib/Plotting/plotTraceWithStd_perTime.py b/HPOlib/Plotting/plotTraceWithStd_perTime.py index 432d77e2..e02249b6 100644 --- a/HPOlib/Plotting/plotTraceWithStd_perTime.py +++ b/HPOlib/Plotting/plotTraceWithStd_perTime.py @@ -35,13 +35,8 @@ def plot_optimization_trace(trial_list, name_list, times_list, optimum=0, title="", log=True, save="", y_max=0, y_min=0, scale_std=1): - markers = itertools.cycle(['o', 's', 'x']) - colors = itertools.cycle(['b', 'Navy', 'RoyalBlue', - 'g', 'DarkGreen', 'SpringGreen', - 'r', 'Maroon', 'LightCoral']) - colors_alt = itertools.cycle(['Navy', 'RoyalBlue', - 'DarkGreen', 'SpringGreen', - 'Maroon', 'LightCoral']) + markers = plot_util.get_plot_markers() + colors = plot_util.get_plot_colors() linestyles = itertools.cycle(['-']) size = 1 diff --git a/HPOlib/Plotting/plot_util.py b/HPOlib/Plotting/plot_util.py index fc3ecce0..4f148bd4 100644 --- a/HPOlib/Plotting/plot_util.py +++ b/HPOlib/Plotting/plot_util.py @@ -17,6 +17,7 @@ # along with this program. If not, see . import cPickle +import itertools import os import numpy as np import sys @@ -25,6 +26,23 @@ __contact__ = "automl.org" +def get_plot_markers(): + return itertools.cycle(['o', 's', 'x', '^']) + + +def get_plot_colors(): + # color brewer, 2nd qualitative 9 color scheme (http://colorbrewer2.org/) + return itertools.cycle(["#e41a1c", # Red + "#377eb8", # Blue + "#4daf4a", # Green + "#984ea3", # Purple + "#ff7f00", # Orange + "#ffff33", # Yellow + "#a65628", # Brown + "#f781bf", # Pink + "#999999"]) # Grey + + def read_pickles(name_list, pkl_list, cut=sys.maxint): best_dict = dict() idx_dict = dict() From 28cdf58a089d3a5a6b8fc858d668d3cd2d7c2a24 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Mon, 24 Mar 2014 18:14:59 +0100 Subject: [PATCH 130/139] REFACTOR: cleaned up interface of tex table --- HPOlib/Plotting/doAllPlots.py | 4 +--- HPOlib/Plotting/generateTexTable.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/HPOlib/Plotting/doAllPlots.py b/HPOlib/Plotting/doAllPlots.py index b196577a..42132d2b 100644 --- a/HPOlib/Plotting/doAllPlots.py +++ b/HPOlib/Plotting/doAllPlots.py @@ -128,9 +128,7 @@ def _generate_tex_table(pkl_list, name_list, save="", cut=sys.maxint, try: os.chdir(plotting_dir) import generateTexTable - table = generateTexTable.main(pkl_list, name_list, cut) - with open(save, "w") as fh: - fh.write(table) + generateTexTable.main(pkl_list, name_list, save, cut) os.chdir(cur_dir) sys.stdout.write("passed\n") except Exception, e: diff --git a/HPOlib/Plotting/generateTexTable.py b/HPOlib/Plotting/generateTexTable.py index 4e2bf201..76cff0ec 100644 --- a/HPOlib/Plotting/generateTexTable.py +++ b/HPOlib/Plotting/generateTexTable.py @@ -71,7 +71,7 @@ \\end{document} """ -def main(pkl_list, name_list, cut=sys.maxint): +def main(pkl_list, name_list, save, cut=sys.maxint): tex = StringIO() result_values = OrderedDict([(name[0], dict()) for name in name_list]) @@ -114,8 +114,12 @@ def main(pkl_list, name_list, cut=sys.maxint): tex.seek(0) table = tex.getvalue() - print table - return table + + if save != "": + with open(save, "w") as fh: + fh.write(table) + else: + print table if __name__ == "__main__": @@ -126,6 +130,8 @@ def main(pkl_list, name_list, cut=sys.maxint): parser.add_argument("-c", "--cut", dest="cut", default=sys.maxint, type=int, help="Only consider that many evaluations") + parser.add_argument("-s", "--save", dest="save", default="", + help="Where to save plot instead of showing it?") args, unknown = parser.parse_known_args() # TODO-list: # 1. Add statistical relevance @@ -135,6 +141,6 @@ def main(pkl_list, name_list, cut=sys.maxint): # 4. Determine the experiment name and number of evaluations pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown) - main(pkl_list_main, name_list_main, args.cut) + main(pkl_list_main, name_list_main, args.save, args.cut) From 11f7793063707d7ee174c2fb91b0023522b5d945 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Mon, 24 Mar 2014 18:22:20 +0100 Subject: [PATCH 131/139] FIX: #28 > ```python > # TODO: We do not have any old runs anymore. DELETE this! > if args.restore: > ``` > Is there any reason to keep this? No! --- HPOlib/wrapping.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 07d8ba48..904bbcba 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -204,29 +204,7 @@ def main(): title=args.title) trials.optimizer = optimizer_version - # TODO: We do not have any old runs anymore. DELETE this! if args.restore: - # Old versions did store NaNs instead of the worst possible result for - # crashed runs in the instance_results. In order to be able to load - # these files, these NaNs are replaced - for i, trial in enumerate(trials.instance_order): - _id, fold = trial - instance_result = trials.get_trial_from_id(_id)['instance_results'][fold] - if not np.isfinite(instance_result): - # Make sure that we do delete the last job which was running but - # did not finish - if i == len(trials.instance_order) - 1 and \ - len(trials.cv_starttime) != len(trials.cv_endtime): - # The last job obviously did not finish correctly, do not - # replace it - pass - else: - trials.get_trial_from_id(_id)['instance_results'][fold] = \ - config.getfloat('HPOLIB', 'result_on_terminate') - # Pretty sure we need this here: - trials.get_trial_from_id(_id)['instance_status'][fold] = \ - Experiment.BROKEN_STATE - #noinspection PyBroadException try: restored_runs = optimizer_module.restore(config=config, From 30f4ca6053e6b57ec54237c1db1f3a930578c489 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Mon, 24 Mar 2014 18:26:26 +0100 Subject: [PATCH 132/139] REFACTOR: removed setenv from wrapping.py #28 --- HPOlib/wrapping.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/HPOlib/wrapping.py b/HPOlib/wrapping.py index 904bbcba..c4bbc886 100644 --- a/HPOlib/wrapping.py +++ b/HPOlib/wrapping.py @@ -169,10 +169,6 @@ def main(): # hold of the new optimizer directory wrapping_dir = os.path.dirname(os.path.realpath(__file__)) - # TODO: We don't need this anymore, if we install HPOlib - # Try adding runsolver to path - os.putenv('PATH', os.environ['PATH'] + ":" + wrapping_dir + "/../runsolver/src/") - # Load optimizer try: optimizer_dir = os.path.dirname(os.path.realpath(optimizer_version)) From 72039bc3a00abb81b1b77d66a8140f427880846d Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Mon, 24 Mar 2014 18:34:11 +0100 Subject: [PATCH 133/139] REFACTOR #28 num cv checked in parse.py now --- HPOlib/config_parser/parse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HPOlib/config_parser/parse.py b/HPOlib/config_parser/parse.py index e468d34b..ee1c6d6f 100644 --- a/HPOlib/config_parser/parse.py +++ b/HPOlib/config_parser/parse.py @@ -57,4 +57,6 @@ def check_config(config): if not config.has_option('HPOLIB', 'function') or \ config.get('HPOLIB', 'function') == '': raise Exception('No function specified in .cfg') + if config.getint('HPOLIB', "number_cv_folds") < 1: + raise Exception("The number of crossvalidation folds must be at least one!") return True From f443690d891d4bc26bd266f8f0a605adc17fe8cc Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Mon, 24 Mar 2014 18:41:02 +0100 Subject: [PATCH 134/139] FIX #43 using runsolver 3.3.4 --- benchmarks/lda_ongrid/config.cfg | 13 ----- .../hyperopt_august2013_mod/space.py | 5 -- benchmarks/lda_ongrid/lda.py | 40 ------------- .../lda_ongrid/random_hyperopt_august2013_mod | 1 - .../lda_ongrid/random_hyperopt_august2014_mod | 1 - .../lda_ongrid/smac_2_06_01-dev/params.pcs | 4 -- .../spearmint_april2013_mod/config.pb | 27 --------- setup.py | 57 ++++++++++--------- 8 files changed, 30 insertions(+), 118 deletions(-) delete mode 100644 benchmarks/lda_ongrid/config.cfg delete mode 100644 benchmarks/lda_ongrid/hyperopt_august2013_mod/space.py delete mode 100644 benchmarks/lda_ongrid/lda.py delete mode 120000 benchmarks/lda_ongrid/random_hyperopt_august2013_mod delete mode 120000 benchmarks/lda_ongrid/random_hyperopt_august2014_mod delete mode 100644 benchmarks/lda_ongrid/smac_2_06_01-dev/params.pcs delete mode 100644 benchmarks/lda_ongrid/spearmint_april2013_mod/config.pb diff --git a/benchmarks/lda_ongrid/config.cfg b/benchmarks/lda_ongrid/config.cfg deleted file mode 100644 index 6c245bdb..00000000 --- a/benchmarks/lda_ongrid/config.cfg +++ /dev/null @@ -1,13 +0,0 @@ -[SMAC] -p = smac_2_06_01-dev/params.pcs - -[TPE] -space = space.py - -[SPEARMINT] -config = config.pb - -[HPOLIB] -function = python ../lda.py -number_of_jobs = 50 -result_on_terminate = 1000000000 diff --git a/benchmarks/lda_ongrid/hyperopt_august2013_mod/space.py b/benchmarks/lda_ongrid/hyperopt_august2013_mod/space.py deleted file mode 100644 index 3b8ead30..00000000 --- a/benchmarks/lda_ongrid/hyperopt_august2013_mod/space.py +++ /dev/null @@ -1,5 +0,0 @@ -from hyperopt import hp - -space = {'Kappa': hp.quniform('Kappa', 0, 5, 1), - 'Tau': hp.quniform('Tau', 0, 5, 1), - 'S': hp.quniform('S', 0, 7, 1)} diff --git a/benchmarks/lda_ongrid/lda.py b/benchmarks/lda_ongrid/lda.py deleted file mode 100644 index c21a1125..00000000 --- a/benchmarks/lda_ongrid/lda.py +++ /dev/null @@ -1,40 +0,0 @@ -## -# wrapping: A program making it easy to use hyperparameter -# optimization software. -# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -import HPOlib.benchmark_util as benchmark_util -import HPOlib.benchmark_functions as benchmark_functions - -__authors__ = ["Katharina Eggensperger", "Matthias Feurer"] -__contact__ = "automl.org" -__credits__ = ["Jasper Snoek", "Ryan P. Adams", "Hugo Larochelle"] - - -def main(params, ret_time=False, **kwargs): - print 'Params: ', params - y = benchmark_functions.save_lda_on_grid(params, ret_time=ret_time, **kwargs) - print 'Result: ', y - return y - - -if __name__ == "__main__": - args, params = benchmark_util.parse_cli() - result = main(params, ret_time=False, **args) - duration = main(params, ret_time=True) - print "Result for ParamILS: %s, %f, 1, %f, %d, %s" % \ - ("SAT", abs(duration), result, -1, str(__file__)) \ No newline at end of file diff --git a/benchmarks/lda_ongrid/random_hyperopt_august2013_mod b/benchmarks/lda_ongrid/random_hyperopt_august2013_mod deleted file mode 120000 index c4f9ce32..00000000 --- a/benchmarks/lda_ongrid/random_hyperopt_august2013_mod +++ /dev/null @@ -1 +0,0 @@ -hyperopt_august2013_mod \ No newline at end of file diff --git a/benchmarks/lda_ongrid/random_hyperopt_august2014_mod b/benchmarks/lda_ongrid/random_hyperopt_august2014_mod deleted file mode 120000 index c4f9ce32..00000000 --- a/benchmarks/lda_ongrid/random_hyperopt_august2014_mod +++ /dev/null @@ -1 +0,0 @@ -hyperopt_august2013_mod \ No newline at end of file diff --git a/benchmarks/lda_ongrid/smac_2_06_01-dev/params.pcs b/benchmarks/lda_ongrid/smac_2_06_01-dev/params.pcs deleted file mode 100644 index c3551e94..00000000 --- a/benchmarks/lda_ongrid/smac_2_06_01-dev/params.pcs +++ /dev/null @@ -1,4 +0,0 @@ -Kappa [0, 5] [0]i -Tau [0, 5] [0]i -S [0, 7] [0]i - diff --git a/benchmarks/lda_ongrid/spearmint_april2013_mod/config.pb b/benchmarks/lda_ongrid/spearmint_april2013_mod/config.pb deleted file mode 100644 index 0dff3cdb..00000000 --- a/benchmarks/lda_ongrid/spearmint_april2013_mod/config.pb +++ /dev/null @@ -1,27 +0,0 @@ -language: PYTHON -name: "HPOlib.cv" - -variable { - name: "Kappa" - type: INT - size: 1 - min: 0 - max: 5 -} - -variable { - name: "Tau" - type: INT - size: 1 - min: 0 - max: 5 -} - -variable { - name: "S" - type: INT - size: 1 - min: 0 - max: 7 -} - diff --git a/setup.py b/setup.py index bfb7ba46..9e0b7417 100644 --- a/setup.py +++ b/setup.py @@ -40,11 +40,11 @@ def download_source(download_url, md5, save_as): try: urllib.urlretrieve(download_url, filename=save_as) except Exception, e: - print("Error downloading %s: %s" % (download_url, e)) + sys.stdout.write("Error downloading %s: %s\n" % (download_url, e)) return False md5_new = hashlib.md5(open(save_as).read()).hexdigest() if md5_new != md5: - print "md5 checksum has changed: %s to %s" % (md5, md5_new) + sys.stdout.write("md5 checksum has changed: %s to %s\n" % (md5, md5_new)) return False return True @@ -55,7 +55,7 @@ def extract_source(fn_name, extract_as): tfile = tarfile.open(fn_name) tfile.extractall(extract_as) except Exception, e: - print("Error occurred during extraction: %s" % e) + sys.stdout.write("Error occurred during extraction: %s\n" % e) return False return True return False @@ -103,19 +103,19 @@ def _check_for_runsolver(self): if os.access(p, flags) and not os.path.isdir(p): if not os.path.isdir(os.path.join(os.getcwd(), 'runsolver/src/')): os.mkdir(os.path.join(os.getcwd(), 'runsolver/src/')) - print "Copy runsolver from %s to runsolver/src/runsolver" % p + sys.stdout.write("Copy runsolver from %s to runsolver/src/runsolver\n" % p) shutil.copy(p, os.path.join(os.getcwd(), 'runsolver/src/runsolver')) return p return False def _build(self, build_dir): - print("Building runsolver") + sys.stdout.write("Building runsolver\n") cur_pwd = os.getcwd() os.chdir(build_dir) try: subprocess.check_call("make") except subprocess.CalledProcessError, e: - print "Error during building runsolver: %s" % e + sys.stdout.write("Error while building runsolver: %s\n" % e) os.chdir(cur_pwd) return False os.chdir(cur_pwd) @@ -141,8 +141,8 @@ def _copy_and_download_optimizer(self, optimizer_dir, optimizer_name, optimizer_ def run(self): # RUNSOLVER STUFF here_we_are = os.getcwd() - runsolver_tar_name = "runsolver-3.3.2.tar.bz2" - runsolver_name = "runsolver-3.3.2" + runsolver_tar_name = "runsolver-3.3.4.tar.bz2" + runsolver_name = "runsolver-3.3.4" if sys.version_info < (2, 7, 0) or sys.version_info >= (2, 8, 0): sys.stderr.write("HPOlib requires Python 2.7.0\n") sys.exit(-1) @@ -152,32 +152,35 @@ def run(self): runsolver_path = self._check_for_runsolver() if not runsolver_path: - sys.stdout.write("[INFO] 'runsolver' not found, so we try to install it") + sys.stdout.write("[INFO] 'runsolver' not found, so we try to install it\n") runsolver_needs_to_be_installed = True if runsolver_needs_to_be_installed: - sys.stdout.write("Downloading runsolver from %s%s, saving to %s/%s" % + sys.stdout.write("Downloading runsolver from %s%s, saving to %s/%s\n" % ('http://www.cril.univ-artois.fr/~roussel/runsolver/', runsolver_tar_name, os.getcwd(), runsolver_tar_name)) downloaded = download_source(download_url='http://www.cril.univ-artois.fr/~roussel/runsolver/%s' % runsolver_tar_name, - md5="62d139a6d2b1b376eb9ed59ccb5d6726", + md5="5a9511266489c87f4a276b9e54ea4082", save_as=os.path.join(here_we_are, runsolver_tar_name)) if runsolver_needs_to_be_installed and downloaded: - print("Extracting runsolver to %s" % os.path.join(here_we_are, runsolver_name)) + sys.stdout.write("Extracting runsolver to %s\n" % os.path.join(here_we_are, runsolver_name)) extracted = extract_source(fn_name=os.path.join(here_we_are, runsolver_tar_name), extract_as=here_we_are) if runsolver_needs_to_be_installed and extracted: - print("Building runsolver") - built = self._build(build_dir=os.path.join(here_we_are, "runsolver", "src")) - if not built: - print "Try a second time and replace '/usr/include/asm/unistd.h' with '/usr/include/unistd.h'" - call = "sed -i 's/\/usr\/include\/asm\/unistd/\/usr\/include\/unistd/g' runsolver/src/Makefile" - try: - subprocess.check_call(call, shell=True) - except subprocess.CalledProcessError, e: - print "Replacing did not work: %s" % e + sys.stdout.write("Building runsolver\n") + sys.stdout.write("Replace a line, which seems to cause trouble on 32 bit systems'\n") + call = "sed -i 's|gnu/asm/unistd_$(WSIZE).h|gnu/asm/unistd_$(WSIZE).h /usr/include/i386-linux-gnu/asm/unistd_$(WSIZE).h|' runsolver/src/Makefile" + try: + subprocess.check_call(call, shell=True) + except subprocess.CalledProcessError, e: + sys.stdout.write("Replacing did not work: %s\n" % e) + md5_new = hashlib.md5(open("runsolver/src/Makefile").read()).hexdigest() + runsolver_md5 = "4870722e47a6f74d5167376d371f1730" + if md5_new != runsolver_md5: + sys.stdout.write("md5 checksum has changed: %s to %s, not compiling runsolver\n" % (runsolver_md5, md5_new)) + else: built = self._build(build_dir=os.path.join(here_we_are, "runsolver", "src")) # COPY/DOWNLOAD OPTIMIZER TO ROOT FOLDER @@ -208,7 +211,6 @@ def run(self): # if anyone knows a better way, feel free to change install.do_egg_install(self) - # Give detailed output to user if not tpe or not smac or not spearmint: sys.stderr.write("[ERROR] Something went wrong while copying and downloading optimizers." + @@ -224,13 +226,14 @@ def run(self): "mv spearmint_april2013_mod_src spearmint/ \n\n" + "Thank You!\n") if runsolver_needs_to_be_installed and not built: - print "[ERROR] Please install runsolver on your own! You can download it from:\n%s%s" % \ - ('http://www.cril.univ-artois.fr/~roussel/runsolver/%s', runsolver_tar_name) - print "[ERROR] Be sure 'runsolver' can be found during runtime in your $PATH variable!" + sys.stdout.write("[ERROR] Please install runsolver on your own! You can download it from:\n%s%s\n" % \ + ('http://www.cril.univ-artois.fr/~roussel/runsolver/%s', runsolver_tar_name)) + sys.stdout.write("[ERROR] Be sure 'runsolver' can be found during runtime in your $PATH variable!\n") if not runsolver_needs_to_be_installed and not built: - print "[INFO] 'runsolver' has been found on this system and was copied from: %s" % runsolver_path + sys.stdout.write("[INFO] 'runsolver' has been found on this system and was copied from: %s\n" % + runsolver_path) if built: - print "'runsolver' has been downloaded and installed" + sys.stdout.write("'runsolver' has been downloaded and installed\n") setup( name='HPOlib', From c47f0da64680330fbead13a2a83c60d9d2ab4e07 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Mon, 24 Mar 2014 18:43:19 +0100 Subject: [PATCH 135/139] REFACTOR: #20 removed DEFAULT handling code --- HPOlib/cv.py | 6 ------ HPOlib/wrapping_util.py | 9 --------- 2 files changed, 15 deletions(-) diff --git a/HPOlib/cv.py b/HPOlib/cv.py index 4b3bc07f..688e79b1 100644 --- a/HPOlib/cv.py +++ b/HPOlib/cv.py @@ -57,12 +57,6 @@ def do_cv(params, folds=10): optimizer = get_optimizer() cfg = load_experiment_config_file() - # Now evaluate $fold times - # TODO This is a bad idea, no one replaces this by the worst possible - # value later on - if folds < 1: - return np.NaN - # Store the results to hand them back to tpe and spearmint results = [] diff --git a/HPOlib/wrapping_util.py b/HPOlib/wrapping_util.py index 697708fb..30536998 100644 --- a/HPOlib/wrapping_util.py +++ b/HPOlib/wrapping_util.py @@ -216,8 +216,6 @@ def parse_config_values_from_unknown_arguments(unknown_arguments, config): for key in config.options(section): further_possible_command_line_arguments.append("--" + section + ":" + key) - for key in config.defaults(): - further_possible_command_line_arguments.append("--DEFAULT:" + key) parser = ArgumentParser() for argument in further_possible_command_line_arguments: @@ -239,13 +237,6 @@ def config_with_cli_arguments(config, config_overrides): def save_config_to_file(file_handle, config, write_nones=True): - if len(config.defaults()) > 0: - file_handle.write("[DEFAULT]\n") - for key in config.defaults(): - if (config.get("DEFAULT", key) is None and write_nones) or \ - config.get("DEFAULT", key) is not None: - file_handle.write(key + " = " + config.get("DEFAULT", key) + "\n") - for section in config.sections(): file_handle.write("[" + section + "]\n") for key in config.options(section): From 5782e9a337b3e36fa1cbca516c1248a18244e268 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 25 Mar 2014 09:56:06 +0100 Subject: [PATCH 136/139] Changed version to 0.1.0rc1 and Development Status :: 3 - Alpha --- HPOlib/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HPOlib/__init__.py b/HPOlib/__init__.py index e14fe892..64ca9bc7 100644 --- a/HPOlib/__init__.py +++ b/HPOlib/__init__.py @@ -1,4 +1,4 @@ __authors__ = ["Katharina Eggensperger", "Matthias Feurer"] __contact__ = "automl.org" -__version__ = "0.0.1" +__version__ = "0.1.0rc1" diff --git a/setup.py b/setup.py index 9e0b7417..13a9f8e2 100644 --- a/setup.py +++ b/setup.py @@ -265,7 +265,7 @@ def run(self): cmdclass={'install': AdditionalInstall}, classifiers=[ 'Programming Language :: Python :: 2.7', - 'Development Status :: 2 - Pre-Alpha', + 'Development Status :: 3 - Alpha', 'Natural Language :: English', 'Environment :: Console', 'Intended Audience :: Developers', From 4762541ee84ffc9cd2c91089a1faefaa4b3957cc Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 25 Mar 2014 10:22:36 +0100 Subject: [PATCH 137/139] ENHANCEMENT: Adding Travis --- .travis.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..b007c1fc --- /dev/null +++ b/.travis.yml @@ -0,0 +1,12 @@ +language: python +python: + - "2.7" +# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors +install: + - easy_install -U distribute + - pip install numpy + - pip install scipy==0.13.2 + - pip install matplotlib + - python setup.py install +# command to run tests, e.g. python setup.py test +script: python setup.py test From a6a638b073b2dff9240d722078c9b57dd9772c59 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 25 Mar 2014 10:34:35 +0100 Subject: [PATCH 138/139] Get Travis working with Scipy --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index b007c1fc..44a7ae4a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,11 @@ language: python python: - "2.7" # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors + +before_install: + - sudo apt-get update -qq + - sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran + install: - easy_install -U distribute - pip install numpy From 0da085ecc57bf9b18ad58f2f4d1e552969a2de03 Mon Sep 17 00:00:00 2001 From: KEggensperger Date: Tue, 25 Mar 2014 10:47:26 +0100 Subject: [PATCH 139/139] Travis --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 44a7ae4a..91252f78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,7 @@ python: # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors before_install: - - sudo apt-get update -qq - - sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran + - sudo apt-get install -q libatlas3gf-base libatlas-dev liblapack-dev gfortran install: - easy_install -U distribute