Merge branch 'development'

.gitignore

@@ -1,9 +1,7 @@
 *.py[cod]
 
 # Optimizers
-HPOlib/optimizers/smac_2_06_01-dev/*
-HPOlib/optimizers/spearmint_april2013_mod/*
-HPOlib/optimizers/hyperopt_august2013_mod/*
+*_src*
 
 # Runsolver
 runsolver/src/*

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "optimizers/spearmint/spearmint_gitfork_mod_src"]
+	path = optimizers/spearmint/spearmint_gitfork_mod_src
+	url = https://github.com/automl/spearmint.git

HPOlib/Experiment.py

@@ -19,8 +19,6 @@
 import cPickle
 import logging
 import os
-import scipy
-from scipy.stats.distributions import wrapcauchy_gen
 import sys
 import tempfile
 

HPOlib/Locker.py

@@ -61,7 +61,7 @@ def unlock(self, filename):
         if self.locks[filename] == 1:
             success = safe_delete('%s.lock' % (filename))
             if not success:
-                logger.log("Could not unlock file: %s.\n", filename)
+                logger.error("Could not unlock file: %s.\n", filename)
             del self.locks[filename]
             return success
         else:

HPOlib/Plotting/doAllPlots.py

@@ -227,23 +227,16 @@ def main():
         _box_whisker(pkl_list=pkl_list, name_list=name_list, save=tmp_save,
                      log=log, cut=args.cut)
 
-        # statistics
-        if save_dir is not "":
-            tmp_save = os.path.join(save_dir, "statistics_%s.txt" % time_str)
-        else:
-            tmp_save = save_dir
-        sys.stdout.write("statistics.py ... %s ..." % tmp_save)
-        _statistics(pkl_list=pkl_list, name_list=name_list, save=tmp_save,
-                    log=log, cut=args.cut)
-
         # LaTeX table
         if save_dir is not "":
             tmp_save = os.path.join(save_dir, "table_%s.tex" % time_str)
         else:
             tmp_save = save_dir
         sys.stdout.write("generateTexTable.py ... %s ..." % tmp_save)
-        _generate_tex_table(pkl_list=pkl_list, name_list=name_list,
+        ret = _generate_tex_table(pkl_list=pkl_list, name_list=name_list,
                             save=tmp_save, log=log, cut=args.cut)
+        if ret is not None:
+            print ret
 
 
     # We can always plot this
@@ -256,6 +249,15 @@ def main():
     _optimizer_overhead(pkl_list=pkl_list, name_list=name_list, save=tmp_save,
                         log=log, cut=args.cut)
 
+    # statistics
+    if save_dir is not "":
+        tmp_save = os.path.join(save_dir, "statistics_%s.txt" % time_str)
+    else:
+        tmp_save = save_dir
+    sys.stdout.write("statistics.py ... %s ..." % tmp_save)
+    _statistics(pkl_list=pkl_list, name_list=name_list, save=tmp_save,
+                log=log, cut=args.cut)
+
     # Error Trace with Std
     if save_dir is not "":
         tmp_save = os.path.join(save_dir, "TraceWithStd_perEval_%s.%s" % (time_str, args.file))

HPOlib/Plotting/generateTexTable.py

@@ -31,6 +31,7 @@
     jinja2 = ""
 
 from HPOlib.Plotting import plot_util
+from HPOlib import wrapping_util
 
 __authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
 __contact__ = "automl.org"
@@ -42,7 +43,7 @@
 \\usepackage[landscape]{geometry}
 \\usepackage{multirow}           % import command \multicolmun
 \\usepackage{tabularx}           % Convenient table formatting
-\\usepackage{booktabs}           % provides \toprule, \midrule and \bottomrule
+\\usepackage{booktabs}           % provides \\toprule, \midrule and \\bottomrule
 
 \\begin{document}
 
@@ -63,43 +64,57 @@
 {{ experiment }} & {{ evals }}
 {%- for name in result_values -%}
 {%- set results = result_values[name] -%}
-{{ ' & ' }}{{ results['mean']|round(3, 'floor') }}$\\pm${{ results['std']|round(3, 'floor')}} & {{ results['min']|round(3, 'floor') }}
-{%- endfor %} \\\\
+{{ ' & ' }}{% if results['mean_best'] == True %}\\textbf{ {%- endif %}{{results['mean']|round(3, 'floor') }}{% if results['mean_best'] == True %}}{% endif %}$\\pm${{ results['std']|round(3, 'floor')}} & {{results['min']|round(3, 'floor') }}{%- endfor %} \\\\
 \\bottomrule
 \\end{tabularx}
 \\end{table}
 \\end{document}
 """
 
-def main(pkl_list, name_list, save, cut=sys.maxint):
+def main(pkl_list, name_list, save="", cut=sys.maxint,
+         template_string=template_string, experiment_name="Name",
+         num_evals="\\#eval"):
+    pickles = plot_util.load_pickles(name_list, pkl_list)
+    best_dict, idx_dict, keys = plot_util.get_best_dict(name_list, pickles, cut)
+    return generate_tex_template(best_dict, name_list,
+                          template_string=template_string, save=save,
+                          num_evals=num_evals, experiment_name=experiment_name)
+
+
+def generate_tex_template(best_dict, name_list, save="",
+         template_string=template_string, experiment_name="Name",
+         num_evals="\\#eval"):
     tex = StringIO()
     result_values = OrderedDict([(name[0], dict()) for name in name_list])
 
-    best_dict, idx_dict, keys = plot_util.read_pickles(name_list, pkl_list, cut)
+    means = [np.mean(best_dict[name]) for name in result_values]
+    stds = [np.std(best_dict[name]) for name in result_values]
+    mins = [np.min(best_dict[name]) for name in result_values]
+    maxs = [np.max(best_dict[name]) for name in result_values]
 
     for name in result_values:
         values = result_values[name]
 
         values["mean"] = np.mean(best_dict[name])
-        values["mean_bold"] = False
-        values["mean_italic"] = False
+        values["mean_best"] = True if \
+            wrapping_util.float_eq(values["mean"], min(means)) else False
 
         values["std"] = np.std(best_dict[name])
-        values["std_bold"] = False
-        values["std_italic"] = False
+        values["std_best"] = True if \
+            wrapping_util.float_eq(values["std"], min(stds)) else False
 
         values["min"] = np.min(best_dict[name])
-        values["min_bold"] = False
-        values["min_italic"] = False
+        values["min_best"] = True if\
+            wrapping_util.float_eq(values["min"], min(mins)) else False
 
-        values["max"] = np.min(best_dict[name])
-        values["max_bold"] = False
-        values["max_italic"] = False
+        values["max"] = np.max(best_dict[name])
+        values["max_best"] = True if\
+            wrapping_util.float_eq(values["max"], min(maxs)) else False
 
     if jinja2:
         template = Template(template_string)
         tex.write(template.render(result_values=result_values,
-                                  experiment="Name", evals="\\#evals"))
+                                  experiment=experiment_name, evals=num_evals))
     else:
         tex.write("Name & #evals")
         for name in result_values:
@@ -119,7 +134,7 @@ def main(pkl_list, name_list, save, cut=sys.maxint):
         with open(save, "w") as fh:
             fh.write(table)
     else:
-        print table
+        return table
 
 
 if __name__ == "__main__":

HPOlib/Plotting/plotTrace_perExp.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+
+##
+# wrapping: A program making it easy to use hyperparameter
+# optimization software.
+# Copyright (C) 2013 Katharina Eggensperger and Matthias Feurer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from argparse import ArgumentParser
+import cPickle
+import itertools
+import sys
+
+from matplotlib.pyplot import tight_layout, figure, subplots_adjust, subplot, savefig, show
+import matplotlib.gridspec
+import numpy as np
+
+from HPOlib.Plotting import plot_util
+
+__authors__ = ["Katharina Eggensperger", "Matthias Feurer"]
+__contact__ = "automl.org"
+
+
+def plot_optimization_trace_cv(trial_list, name_list, optimum=0, title="",
+                               log=True, save="", y_max=0, y_min=0):
+    markers =plot_util.get_plot_markers()
+    colors = plot_util.get_plot_colors()
+    linestyles = itertools.cycle(['-'])
+    size = 1
+
+    ratio = 5
+    gs = matplotlib.gridspec.GridSpec(ratio, 1)
+    fig = figure(1, dpi=100)
+    fig.suptitle(title, fontsize=16)
+    ax1 = subplot(gs[0:ratio, :])
+    ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
+    min_val = sys.maxint
+    max_val = -sys.maxint
+    max_trials = 0
+
+    fig.suptitle(title, fontsize=16)
+
+    # Plot the average error and std
+    for i in range(len(name_list)):
+        m = markers.next()
+        c = colors.next()
+        l = linestyles.next()
+        leg = False
+        for tr in trial_list[i]:
+            if log:
+                tr = np.log10(tr)
+            x = range(1, len(tr)+1)
+            y = tr
+            if not leg:
+                ax1.plot(x, y, color=c, linewidth=size, linestyle=l, label=name_list[i][0])
+                leg = True
+            ax1.plot(x, y, color=c, linewidth=size, linestyle=l)
+            min_val = min(min_val, min(tr))
+            max_val = max(max_val, max(tr))
+            max_trials = max(max_trials, len(tr))
+
+    # Maybe plot on logscale
+    ylabel = ""
+
+    if log:
+        ax1.set_ylabel("log10(Minfunction value)" + ylabel)
+    else:
+        ax1.set_ylabel("Minfunction value" + ylabel)
+
+    # Descript and label the stuff
+    leg = ax1.legend(loc='best', fancybox=True)
+    leg.get_frame().set_alpha(0.5)
+    ax1.set_xlabel("#Function evaluations")
+
+    if y_max == y_min:
+         # Set axes limits
+        ax1.set_ylim([min_val-0.1*abs((max_val-min_val)), max_val+0.1*abs((max_val-min_val))])
+    else:
+        ax1.set_ylim([y_min, y_max])
+    ax1.set_xlim([0, max_trials + 1])
+
+    tight_layout()
+    subplots_adjust(top=0.85)
+    if save != "":
+        savefig(save, dpi=100, facecolor='w', edgecolor='w',
+                orientation='portrait', papertype=None, format=None,
+                transparent=False, bbox_inches="tight", pad_inches=0.1)
+    else:
+        show()
+
+
+def main(pkl_list, name_list, autofill, optimum=0, save="", title="", log=False,
+         y_min=0, y_max=0):
+
+    trial_list = list()
+    for i in range(len(pkl_list)):
+        tmp_trial_list = list()
+        max_len = -sys.maxint
+        for pkl in pkl_list[i]:
+            fh = open(pkl, "r")
+            trials = cPickle.load(fh)
+            fh.close()
+
+            trace = plot_util.get_Trace_cv(trials)
+            tmp_trial_list.append(trace)
+            max_len = max(max_len, len(trace))
+        trial_list.append(list())
+        for tr in tmp_trial_list:
+        #    if len(tr) < max_len:
+        #        tr.extend([tr[-1] for idx in range(abs(max_len - len(tr)))])
+            trial_list[-1].append(np.array(tr))
+
+    plot_optimization_trace_cv(trial_list, name_list, optimum, title=title, log=log,
+                            save=save, y_min=y_min, y_max=y_max)
+
+    if save != "":
+        sys.stdout.write("Saved plot to " + save + "\n")
+    else:
+        sys.stdout.write("..Done\n")
+
+if __name__ == "__main__":
+    prog = "python plotTraceWithStd.py WhatIsThis <oneOrMorePickles> [WhatIsThis <oneOrMorePickles>]"
+    description = "Plot a Trace with std for multiple experiments"
+
+    parser = ArgumentParser(description=description, prog=prog)
+
+    # Options for specific benchmarks
+    parser.add_argument("-o", "--optimum", type=float, dest="optimum",
+                        default=0, help="If not set, the optimum is supposed to be zero")
+
+    # Options which are available only for this plot
+    parser.add_argument("-a", "--autofill", action="store_true", dest="autofill",
+                        default=False, help="Fill trace automatically")
+
+    # General Options
+    parser.add_argument("-l", "--log", action="store_true", dest="log",
+                        default=False, help="Plot on log scale")
+    parser.add_argument("--max", dest="max", type=float,
+                        default=0, help="Maximum of the plot")
+    parser.add_argument("--min", dest="min", type=float,
+                        default=0, help="Minimum of the plot")
+    parser.add_argument("-s", "--save", dest="save",
+                        default="", help="Where to save plot instead of showing it?")
+    parser.add_argument("-t", "--title", dest="title",
+                        default="", help="Optional supertitle for plot")
+
+    args, unknown = parser.parse_known_args()
+
+    sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n")
+
+    pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown)
+
+    main(pkl_list=pkl_list_main, name_list=name_list_main, autofill=args.autofill, optimum=args.optimum,
+         save=args.save, title=args.title, log=args.log, y_min=args.min, y_max=args.max)