Merge pull request #20 from jungtaekkim/0.4.2

0.4.2

.readthedocs.yml

@@ -0,0 +1,15 @@
+version: 2
+
+sphinx:
+  builder: html
+  configuration: docs/conf.py
+
+formats:
+  - pdf
+  - htmlzip
+
+python:
+  version: 3.7
+  install:
+    - requirements: requirements.txt
+    - requirements: requirements-optional.txt

.travis.yml

@@ -1,15 +1,20 @@
 language: python
 dist: xenial
 python:
-  - "2.7"
-  - "3.5"
-  - "3.6"
-  - "3.7"
+  - '2.7'
+  - '3.6'
+  - '3.7'
+  - '3.8'
 install:
   - pip install .
+  - pip install tensorflow
+  - pip install tensorflow-probability
+  - pip install torch
+  - pip install gpytorch
 script:
   - pip install coveralls
-  - coverage run -m pytest
+  - pip install pytest-timeout
+  - coverage run -m pytest tests/common/
   - coveralls
 notifications:
   slack: bayeso:FWBoHH9TMqjKUJWkZxCaTNVE

README.md

@@ -65,9 +65,9 @@ The following `requirements` files include the package list, the purpose of whic
 We test our package in the following versions.
 
 * Python 2.7 (It will be excluded due to the maintenance schedule for Python 2.7, but it is currently tested.)
-* Python 3.5
 * Python 3.6
 * Python 3.7
+* Python 3.8
 
 ## Contributor
 * [Jungtaek Kim](http://mlg.postech.ac.kr/~jtkim/) (POSTECH)

bayeso/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.4.2'

bayeso/bo.py

@@ -1,6 +1,6 @@
 # bo
 # author: Jungtaek Kim (jtkim@postech.ac.kr)
-# last updated: April 21, 2020
+# last updated: April 29, 2020
 
 import numpy as np
 import time
@@ -15,11 +15,15 @@
     cma = None
 import sobol_seq
 
-from bayeso import gp
 from bayeso import acquisition
+from bayeso import constants
+from bayeso.gp import gp
+from bayeso.gp import gp_common
 from bayeso.utils import utils_common
 from bayeso.utils import utils_covariance
-from bayeso import constants
+from bayeso.utils import utils_logger
+
+logger = utils_logger.get_logger('bo')
 
 
 def get_grids(arr_ranges, int_grids):
@@ -110,17 +114,14 @@ def _check_optimizer_method_bo(str_optimizer_method_bo, num_dim, debug):
     assert str_optimizer_method_bo in constants.ALLOWED_OPTIMIZER_METHOD_BO
 
     if str_optimizer_method_bo == 'DIRECT' and directminimize is None: # pragma: no cover
-        if debug:
-            print('[DEBUG] _check_optimizer_method_bo in bo.py: DIRECT is selected, but it is not installed.')
+        logger.warning('DIRECT is selected, but it is not installed.')
         str_optimizer_method_bo = 'L-BFGS-B'
     elif str_optimizer_method_bo == 'CMA-ES' and cma is None: # pragma: no cover
-        if debug:
-            print('[DEBUG] _check_optimizer_method_bo in bo.py: CMA-ES is selected, but it is not installed.')
+        logger.warning('CMA-ES is selected, but it is not installed.')
         str_optimizer_method_bo = 'L-BFGS-B'
     # TODO: It should be checked.
     elif str_optimizer_method_bo == 'CMA-ES' and num_dim == 1: # pragma: no cover
-        if debug:
-            print('[DEBUG] _check_optimizer_method_bo in bo.py: CMA-ES is selected, but a dimension of bounds is 1.')
+        logger.warning('CMA-ES is selected, but a dimension of bounds is 1.')
         str_optimizer_method_bo = 'L-BFGS-B'
     return str_optimizer_method_bo
 
@@ -349,8 +350,7 @@ def _get_initial_sobol(self, int_samples, int_seed=None):
 
         if int_seed is None:
             int_seed = np.random.randint(0, 10000)
-        if self.debug:
-            print('[DEBUG] _get_initial_sobol in bo.py: int_seed', int_seed)
+        if self.debug: logger.debug('seed: {}'.format(int_seed))
         arr_samples = sobol_seq.i4_sobol_generate(self.num_dim, int_samples, int_seed)
         arr_samples = arr_samples * (self.arr_range[:, 1].flatten() - self.arr_range[:, 0].flatten()) + self.arr_range[:, 0].flatten()
         return arr_samples
@@ -404,8 +404,7 @@ def get_initial(self, str_initial_method,
 
         if str_initial_method == 'grid':
             assert fun_objective is not None
-            if self.debug:
-                print('[DEBUG] get_initial in bo.py: int_samples is ignored, because grid is chosen.')
+            if self.debug: logger.debug('int_samples is ignored, because grid is chosen.')
             arr_initials = self._get_initial_grid()
             arr_initials = get_best_acquisition(arr_initials, fun_objective)
         elif str_initial_method == 'uniform':
@@ -416,9 +415,9 @@ def get_initial(self, str_initial_method,
             raise NotImplementedError('get_initial: latin')
         else:
             raise NotImplementedError('get_initial: allowed str_initial_method, but it is not implemented.')
-        if self.debug:
-            print('[DEBUG] get_initial in bo.py: arr_initials')
-            print(arr_initials)
+
+        if self.debug: logger.debug('arr_initials:\n{}'.format(utils_logger.get_str_array(arr_initials)))
+
         return arr_initials
 
     def _optimize_objective(self, fun_acquisition, X_train, Y_train, X_test, cov_X_X, inv_cov_X_X, hyps):
@@ -496,8 +495,7 @@ def _optimize(self, fun_negative_acquisition, str_initial_method, int_samples):
                 )
                 next_point_x = next_point.x
                 list_next_point.append(next_point_x)
-                if self.debug:
-                    print('[DEBUG] _optimize in bo.py: optimized point for acq', next_point_x)
+                if self.debug: logger.debug('acquired sample: {}'.format(utils_logger.get_str_array(next_point_x)))
         elif self.str_optimizer_method_bo == 'DIRECT': # pragma: no cover
             list_bounds = self._get_bounds()
             next_point = directminimize(
@@ -542,8 +540,8 @@ def optimize(self, X_train, Y_train,
         :param str_mlm_method: the name of marginal likelihood maximization method for Gaussian process regression.
         :type str_mlm_method: str., optional
 
-        :returns: acquired example, candidates of acquired examples, acquisition function values over the candidates, covariance matrix by `hyps`, inverse matrix of the covariance matrix, hyperparameters optimized, and execution times. Shape: ((d, ), (`int_samples`, d), (`int_samples`, ), (n, n), (n, n), dict., dict.).
-        :rtype: (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray, dict., dict.)
+        :returns: acquired example and dictionary of information. Shape: ((d, ), dict.).
+        :rtype: (numpy.ndarray, dict.)
 
         :raises: AssertionError
 
@@ -565,7 +563,7 @@ def optimize(self, X_train, Y_train,
 
         time_start = time.time()
 
-        if self.is_normalized:
+        if self.is_normalized and not np.max(Y_train) == np.min(Y_train):
             Y_train = (Y_train - np.min(Y_train)) / (np.max(Y_train) - np.min(Y_train)) * constants.MULTIPLIER_RESPONSE
 
         time_start_gp = time.time()
@@ -578,11 +576,9 @@ def optimize(self, X_train, Y_train,
                 cov_X_X, inv_cov_X_X, hyps = gp.get_optimized_kernel(X_train, Y_train, self.prior_mu, self.str_cov, str_optimizer_method=self.str_optimizer_method_gp, str_modelselection_method=self.str_modelselection_method, debug=self.debug)
                 self.is_optimize_hyps = not _check_hyps_convergence(self.historical_hyps, hyps, self.str_cov, is_fixed_noise)
             else: # pragma: no cover
-                print('[DEBUG] optimize in bo.py: hyps are converged.')
+                if self.debug: logger.debug('hyps converged.')
                 hyps = self.historical_hyps[-1]
-                cov_X_X, inv_cov_X_X, _ = gp.get_kernel_inverse(X_train, hyps, self.str_cov, is_fixed_noise=is_fixed_noise, debug=self.debug)
-        elif str_mlm_method == 'probabilistic': # pragma: no cover
-            raise NotImplementedError('optimize: it will be added.')
+                cov_X_X, inv_cov_X_X, _ = gp_common.get_kernel_inverse(X_train, hyps, self.str_cov, is_fixed_noise=is_fixed_noise, debug=self.debug)
         else: # pragma: no cover
             raise ValueError('optimize: missing condition for str_mlm_method.')
         time_end_gp = time.time()
@@ -600,13 +596,17 @@ def optimize(self, X_train, Y_train,
 
         time_end = time.time()
 
-        times = {
-            'overall': time_end - time_start,
-            'gp': time_end_gp - time_start_gp,
-            'acq': time_end_acq - time_start_acq,
+        dict_info = {
+            'next_points': next_points,
+            'acquisitions': acquisitions,
+            'cov_X_X': cov_X_X,
+            'inv_cov_X_X': inv_cov_X_X,
+            'hyps': hyps,
+            'time_overall': time_end - time_start,
+            'time_gp': time_end_gp - time_start_gp,
+            'time_acq': time_end_acq - time_start_acq,
         }
 
-        if self.debug:
-            print('[DEBUG] optimize in bo.py: time consumed', time_end - time_start, 'sec.')
+        if self.debug: logger.debug('overall time consumed to acquire: {:.4f} sec.'.format(time_end - time_start))
 
-        return next_point, next_points, acquisitions, cov_X_X, inv_cov_X_X, hyps, times
+        return next_point, dict_info

bayeso/constants.py

@@ -9,7 +9,7 @@
 JITTER_COV = 1e-5
 JITTER_LOG = 1e-7
 
-STR_OPTIMIZER_METHOD_GP = 'BFGS'
+STR_OPTIMIZER_METHOD_GP = 'Nelder-Mead'
 STR_OPTIMIZER_METHOD_BO = 'L-BFGS-B'
 STR_GP_COV = 'matern52'
 STR_BO_ACQ = 'ei'
@@ -37,17 +37,18 @@
 TIME_PAUSE = 2.0
 RANGE_SHADE = 1.96
 
-ALLOWED_OPTIMIZER_METHOD_GP = ['BFGS', 'L-BFGS-B', 'DIRECT', 'CMA-ES']
+ALLOWED_OPTIMIZER_METHOD_GP = ['BFGS', 'L-BFGS-B', 'Nelder-Mead', 'DIRECT']
 ALLOWED_OPTIMIZER_METHOD_BO = ['L-BFGS-B', 'DIRECT', 'CMA-ES']
 # INFO: Do not use _ (underscore) in base str_cov.
-ALLOWED_GP_COV_BASE = ['se', 'matern32', 'matern52']
+ALLOWED_GP_COV_BASE = ['eq', 'se', 'matern32', 'matern52']
 ALLOWED_GP_COV_SET = ['set_' + str_cov for str_cov in ALLOWED_GP_COV_BASE]
 ALLOWED_GP_COV = ALLOWED_GP_COV_BASE + ALLOWED_GP_COV_SET
 ALLOWED_BO_ACQ = ['pi', 'ei', 'ucb', 'aei', 'pure_exploit', 'pure_explore']
 ALLOWED_INITIALIZATIONS_BO = ['sobol', 'uniform', 'latin']
 ALLOWED_INITIALIZATIONS_AO = ALLOWED_INITIALIZATIONS_BO + ['grid']
 ALLOWED_MLM_METHOD = ['regular', 'converged']
 ALLOWED_MODELSELECTION_METHOD = ['ml', 'loocv']
+ALLOWED_FRAMEWORK_GP = ['scipy', 'tensorflow', 'gpytorch']
 
 KEYS_INFO_BENCHMARK = ['dim_fun', 'bounds', 'global_minimum_X', 'global_minimum_y']
 

bayeso/covariance.py

@@ -28,7 +28,7 @@ def choose_fun_cov(str_cov, is_grad=False):
     assert isinstance(str_cov, str)
     assert isinstance(is_grad, bool)
 
-    if str_cov == 'se':
+    if str_cov == 'eq' or str_cov == 'se':
         if is_grad:
             fun_cov = grad_cov_se
         else:
@@ -76,7 +76,7 @@ def cov_se(X, Xs, lengthscales, signal):
     if isinstance(lengthscales, np.ndarray):
         assert X.shape[1] == Xs.shape[1] == lengthscales.shape[0]
     else:
-        assert X.shape[0] == Xs.shape[0]
+        assert X.shape[1] == Xs.shape[1]
     dist = scisd.cdist(X / lengthscales, Xs / lengthscales, metric='euclidean')
     cov_ = signal**2 * np.exp(-0.5 * dist**2)
     return cov_
@@ -164,7 +164,7 @@ def cov_matern32(X, Xs, lengthscales, signal):
     if isinstance(lengthscales, np.ndarray):
         assert X.shape[1] == Xs.shape[1] == lengthscales.shape[0]
     else:
-        assert X.shape[0] == Xs.shape[0]
+        assert X.shape[1] == Xs.shape[1]
     assert isinstance(signal, float)
 
     dist = scisd.cdist(X / lengthscales, Xs / lengthscales, metric='euclidean')
@@ -254,7 +254,7 @@ def cov_matern52(X, Xs, lengthscales, signal):
     if isinstance(lengthscales, np.ndarray):
         assert X.shape[1] == Xs.shape[1] == lengthscales.shape[0]
     else:
-        assert X.shape[0] == Xs.shape[0]
+        assert X.shape[1] == Xs.shape[1]
     assert isinstance(signal, float)
 
     dist = scisd.cdist(X / lengthscales, Xs / lengthscales, metric='euclidean')
@@ -353,8 +353,6 @@ def cov_set(str_cov, X, Xs, lengthscales, signal):
     assert str_cov in constants.ALLOWED_GP_COV_BASE
     num_X = X.shape[0]
     num_Xs = Xs.shape[0]
-    num_d_X = X.shape[1]
-    num_d_Xs = Xs.shape[1]
 
     fun_cov = choose_fun_cov(str_cov)
     cov_ = fun_cov(X, Xs, lengthscales, signal)
@@ -406,7 +404,7 @@ def cov_main(str_cov, X, Xs, hyps, same_X_Xs,
         assert num_X == num_Xs
         cov_ += np.eye(num_X) * jitter
 
-    if str_cov == 'se' or str_cov == 'matern32' or str_cov == 'matern52':
+    if str_cov == 'eq' or str_cov == 'se' or str_cov == 'matern32' or str_cov == 'matern52':
         assert len(X.shape) == 2
         assert len(Xs.shape) == 2
         num_d_X = X.shape[1]