From 0a2a3f904ccc60caa3bd24fbfafa48f77ae426b5 Mon Sep 17 00:00:00 2001
From: Thomas L Randall <tlranda@clemson.edu>
Date: Tue, 21 Jun 2022 15:38:46 -0500
Subject: [PATCH 1/5] Allow use of SDV to bias optimizer samples.

Files changed in this commit are referenced from:
https://github.com/deephyper/deephyper/tree/master/deephyper/skopt

An additional change is made to space/space.py to prevent Exception when
probabilities are not explicitly defined by Categorical hyperparameters.
---
 skopt/optimizer/optimizer.py | 517 ++++++++++++++++++++++++++---------
 skopt/space/space.py         | 308 ++++++++++++---------
 2 files changed, 556 insertions(+), 269 deletions(-)

diff --git a/skopt/optimizer/optimizer.py b/skopt/optimizer/optimizer.py
index 397bf282..a8767260 100644
--- a/skopt/optimizer/optimizer.py
+++ b/skopt/optimizer/optimizer.py
@@ -4,13 +4,6 @@
 from numbers import Number
 
 import ConfigSpace as CS
-ccs_active = False
-try:
-    import cconfigspace as CCS
-    ccs_active = True
-except (ImportError, OSError) as a:
-    warnings.warn("CCS could not be loaded and is deactivated: " + str(a), category=ImportWarning)
-
 import numpy as np
 import pandas as pd
 
@@ -24,6 +17,7 @@
 
 from ..acquisition import _gaussian_acquisition
 from ..acquisition import gaussian_acquisition_1D
+from ..acquisition import gaussian_lcb
 from ..learning import GaussianProcessRegressor
 from ..space import Categorical
 from ..space import Space
@@ -44,6 +38,15 @@ def __str__(self):
         return f"The search space is exhausted and cannot sample new unique points!"
 
 
+def boltzman_distribution(x, beta=1):
+    x = np.exp(beta * x)
+    x = x / np.sum(x)
+    return x
+
+
+OBJECTIVE_VALUE_FAILURE = "F"
+
+
 class Optimizer(object):
     """Run bayesian optimisation loop.
 
@@ -85,6 +88,8 @@ class Optimizer(object):
         Number of evaluations of `func` with initialization points
         before approximating it with `base_estimator`. Initial point
         generator can be changed by setting `initial_point_generator`.
+    
+    initial_points : list, default: None
 
     initial_point_generator : str, InitialPointGenerator instance, \
             default: `"random"`
@@ -162,6 +167,9 @@ class Optimizer(object):
         Keeps list of models only as long as the argument given. In the
         case of None, the list has no capped length.
 
+    model_sdv : Model or None, default None
+        A Model from Synthetic-Data-Vault.
+
     Attributes
     ----------
     Xi : list
@@ -183,6 +191,7 @@ def __init__(
         base_estimator="gp",
         n_random_starts=None,
         n_initial_points=10,
+        initial_points=None,
         initial_point_generator="random",
         n_jobs=1,
         acq_func="gp_hedge",
@@ -191,6 +200,9 @@ def __init__(
         model_queue_size=None,
         acq_func_kwargs=None,
         acq_optimizer_kwargs=None,
+        model_sdv=None,
+        sample_max_size=-1,
+        sample_strategy="quantile",
     ):
         args = locals().copy()
         del args["self"]
@@ -203,7 +215,7 @@ def __init__(
         self.acq_func = acq_func
         self.acq_func_kwargs = acq_func_kwargs
 
-        allowed_acq_funcs = ["gp_hedge", "EI", "LCB", "PI", "EIps", "PIps"]
+        allowed_acq_funcs = ["gp_hedge", "EI", "LCB", "qLCB", "PI", "EIps", "PIps"]
         if self.acq_func not in allowed_acq_funcs:
             raise ValueError(
                 "expected acq_func to be in %s, got %s"
@@ -269,17 +281,19 @@ def __init__(
             else:
                 acq_optimizer = "sampling"
 
-        if acq_optimizer not in ["lbfgs", "sampling"]:
+        if acq_optimizer not in ["lbfgs", "sampling", "boltzmann_sampling"]:
             raise ValueError(
                 "Expected acq_optimizer to be 'lbfgs' or "
-                "'sampling', got {0}".format(acq_optimizer)
+                "'sampling' or 'softmax_sampling', got {0}".format(acq_optimizer)
             )
 
-        if not has_gradients(self.base_estimator_) and acq_optimizer != "sampling":
+        if not has_gradients(self.base_estimator_) and not (
+            "sampling" in acq_optimizer
+        ):
             raise ValueError(
-                "The regressor {0} should run with "
-                "acq_optimizer"
-                "='sampling'.".format(type(base_estimator))
+                "The regressor {0} should run with a 'sampling' "
+                "acq_optimizer such as "
+                "'sampling' or 'softmax_sampling'.".format(type(base_estimator))
             )
         self.acq_optimizer = acq_optimizer
 
@@ -291,6 +305,9 @@ def __init__(
         self.n_restarts_optimizer = acq_optimizer_kwargs.get("n_restarts_optimizer", 5)
         self.n_jobs = acq_optimizer_kwargs.get("n_jobs", 1)
         self.filter_duplicated = acq_optimizer_kwargs.get("filter_duplicated", True)
+        self.boltzmann_gamma = acq_optimizer_kwargs.get("boltzmann_gamma", 1)
+        self.boltzmann_psucc = acq_optimizer_kwargs.get("boltzmann_psucc", 0)
+        self.filter_failures = acq_optimizer_kwargs.get("filter_failures", "mean")
         self.acq_optimizer_kwargs = acq_optimizer_kwargs
 
         # Configure search space
@@ -298,33 +315,35 @@ def __init__(
         if type(dimensions) is CS.ConfigurationSpace:
             # Save the config space to do a real copy of the Optimizer
             self.config_space = dimensions
+            self.config_space.seed(self.rng.get_state()[1][0])
 
             if isinstance(self.base_estimator_, GaussianProcessRegressor):
                 raise RuntimeError("GP estimator is not available with ConfigSpace!")
-        elif ccs_active and isinstance(dimensions, CCS.ConfigurationSpace):
-            self.ccs = dimensions
-
-            if isinstance(self.base_estimator_, GaussianProcessRegressor):
-                raise RuntimeError("GP estimator is not available with CCS!")
         else:
 
             # normalize space if GP regressor
             if isinstance(self.base_estimator_, GaussianProcessRegressor):
                 dimensions = normalize_dimensions(dimensions)
 
-        self.space = Space(dimensions)
+        # keep track of the generative model from sdv
+        self.model_sdv = model_sdv
 
-        self._initial_samples = None
+        self.space = Space(dimensions, model_sdv=self.model_sdv)
+
+        self._initial_samples = [] if initial_points is None else initial_points[:]
         self._initial_point_generator = cook_initial_point_generator(
             initial_point_generator
         )
 
         if self._initial_point_generator is not None:
             transformer = self.space.get_transformer()
-            self._initial_samples = self._initial_point_generator.generate(
-                self.space.dimensions,
-                n_initial_points,
-                random_state=self.rng.randint(0, np.iinfo(np.int32).max),
+            self._initial_samples = (
+                self._initial_samples
+                + self._initial_point_generator.generate(
+                    self.space.dimensions,
+                    n_initial_points - len(self._initial_samples),
+                    random_state=self.rng.randint(0, np.iinfo(np.int32).max),
+                )
             )
             self.space.set_transformer(transformer)
 
@@ -353,8 +372,17 @@ def __init__(
         # return same sets of points. Reset to {} at every call to `tell`.
         self.cache_ = {}
 
+        # to avoid duplicated samples
         self.sampled = []
 
+        # for botlzmann strategy
+        self._min_value = 0
+        self._max_value = 0
+
+        # parameters to stabilize the size of the dataset used to fit the surrogate model
+        self._sample_max_size = sample_max_size
+        self._sample_strategy = sample_strategy
+
     def copy(self, random_state=None):
         """Create a shallow copy of an instance of the optimizer.
 
@@ -364,16 +392,10 @@ def copy(self, random_state=None):
             Set the random state of the copy.
         """
 
-        dimens = None
-        if hasattr(self, "config_space"):
-            dimens = self.config_space
-        elif hasattr(self, "ccs"):
-            dimens = self.ccs
-        else:
-            dimens = self.space.dimensions
-
         optimizer = Optimizer(
-            dimensions=dimens,
+            dimensions=self.config_space
+            if hasattr(self, "config_space")
+            else self.space.dimensions,
             base_estimator=self.base_estimator_,
             n_initial_points=self.n_initial_points_,
             initial_point_generator=self._initial_point_generator,
@@ -382,7 +404,11 @@ def copy(self, random_state=None):
             acq_func_kwargs=self.acq_func_kwargs,
             acq_optimizer_kwargs=self.acq_optimizer_kwargs,
             random_state=random_state,
+            model_sdv=self.model_sdv,
+            sample_max_size=self._sample_max_size,
+            sample_strategy=self._sample_strategy,
         )
+
         optimizer._initial_samples = self._initial_samples
 
         optimizer.sampled = self.sampled[:]
@@ -430,7 +456,30 @@ def ask(self, n_points=None, strategy="cl_min"):
             self.sampled.append(x)
             return x
 
-        supported_strategies = ["cl_min", "cl_mean", "cl_max"]
+        if n_points > 0 and (
+            self._n_initial_points > 0 or self.base_estimator_ is None
+        ):
+            if len(self._initial_samples) == 0:
+                X = self._ask_random_points(size=n_points)
+            else:
+                n = min(len(self._initial_samples), n_points)
+                X = self._initial_samples[:n]
+                self._initial_samples = self._initial_samples[n:]
+                X = X + self._ask_random_points(size=(n_points - n))
+            self.sampled.extend(X)
+            return X
+
+        if self.acq_func == "qLCB":
+            strategy = "qLCB"
+
+        supported_strategies = [
+            "cl_min",
+            "cl_mean",
+            "cl_max",
+            "topk",
+            "boltzmann",
+            "qLCB",
+        ]
 
         if not (isinstance(n_points, int) and n_points > 0):
             raise ValueError("n_points should be int > 0, got " + str(n_points))
@@ -443,6 +492,74 @@ def ask(self, n_points=None, strategy="cl_min"):
                 + "got %s" % strategy
             )
 
+        # handle one-shot strategies (topk, softmax)
+        if hasattr(self, "_last_X") and strategy == "topk":
+            idx = np.argsort(self._last_values)[:n_points]
+            next_samples = self._last_X[idx].tolist()
+
+            # to track sampled values and avoid duplicates
+            self.sampled.extend(next_samples)
+
+            return next_samples
+
+        if hasattr(self, "_last_X") and strategy == "boltzmann":
+            values = -self._last_values
+
+            self._min_value = (
+                self._min_value
+                if self._min_value is None
+                else min(values.min(), self._min_value)
+            )
+            self._max_value = (
+                self._max_value
+                if self._max_value is None
+                else max(values.max(), self._max_value)
+            )
+
+            idx = [np.argmax(values)]
+            max_trials = 100
+            trials = 0
+
+            while len(idx) < n_points:
+
+                t = len(self.sampled)
+                if t == 0:
+                    beta = 0
+                else:
+                    beta = (
+                        self.boltzmann_gamma
+                        * np.log(t)
+                        / np.abs(self._max_value - self._min_value)
+                    )
+
+                probs = boltzman_distribution(values, beta)
+
+                new_idx = np.argmax(self.rng.multinomial(1, probs))
+
+                if self.filter_duplicated and new_idx in idx and trials < max_trials:
+                    trials += 1
+                else:
+                    idx.append(new_idx)
+                    self.sampled.append(self._last_X[new_idx].tolist())
+
+            return self._last_X[idx].tolist()
+
+        if hasattr(self, "_est") and self.acq_func == "qLCB":
+            X_s = self.space.rvs(n_samples=self.n_points, random_state=self.rng)
+            X_s = self._filter_duplicated(X_s)
+            X_c = self.space.imp_const.fit_transform(
+                self.space.transform(X_s)
+            )  # candidates
+            mu, std = self._est.predict(X_c, return_std=True)
+            kappa = self.acq_func_kwargs.get("kappa", 1.96)
+            kappas = self.rng.exponential(kappa, size=n_points)
+            X = []
+            for kappa in kappas:
+                values = mu - kappa * std
+                idx = np.argmin(values)
+                X.append(X_s[idx])
+            return X
+
         # Caching the result with n_points not None. If some new parameters
         # are provided to the ask, the cache_ is not used.
         if (n_points, strategy) in self.cache_:
@@ -466,14 +583,16 @@ def ask(self, n_points=None, strategy="cl_min"):
             ti_available = "ps" in self.acq_func and len(opt.yi) > 0
             ti = [t for (_, t) in opt.yi] if ti_available else None
 
+            opt_yi = self._filter_failures(opt.yi)
+
             if strategy == "cl_min":
-                y_lie = np.min(opt.yi) if opt.yi else 0.0  # CL-min lie
+                y_lie = np.min(opt_yi) if opt_yi else 0.0  # CL-min lie
                 t_lie = np.min(ti) if ti is not None else log(sys.float_info.max)
             elif strategy == "cl_mean":
-                y_lie = np.mean(opt.yi) if opt.yi else 0.0  # CL-mean lie
+                y_lie = np.mean(opt_yi) if opt_yi else 0.0  # CL-mean lie
                 t_lie = np.mean(ti) if ti is not None else log(sys.float_info.max)
             else:
-                y_lie = np.max(opt.yi) if opt.yi else 0.0  # CL-max lie
+                y_lie = np.max(opt_yi) if opt_yi else 0.0  # CL-max lie
                 t_lie = np.max(ti) if ti is not None else log(sys.float_info.max)
 
             # Lie to the optimizer.
@@ -489,31 +608,102 @@ def ask(self, n_points=None, strategy="cl_min"):
         return X
 
     def _filter_duplicated(self, samples):
+        """Filter out duplicated values in ``samples``.
+
+        Args:
+            samples (list): the list of samples to filter.
+
+        Returns:
+            list: the filtered list of samples
+        """
 
         if self.filter_duplicated:
             # check duplicated values
 
             if hasattr(self, "config_space"):
                 hps_names = self.config_space.get_hyperparameter_names()
-            elif hasattr(self, "ccs"):
-                hps_names = [x.name for x in self.ccs.hyperparameters]
             else:
                 hps_names = self.space.dimension_names
 
-            df_samples = pd.DataFrame(data=samples, columns=hps_names)
+            df_samples = pd.DataFrame(data=samples, columns=hps_names, dtype="O")
             df_samples = df_samples[~df_samples.duplicated(keep="first")]
 
             if len(self.sampled) > 0:
                 df_history = pd.DataFrame(data=self.sampled, columns=hps_names)
                 df_merge = pd.merge(df_samples, df_history, on=None, how="inner")
-                df_samples = df_samples.append(df_merge)
+                df_samples = pd.concat([df_samples, df_merge])
                 df_samples = df_samples[~df_samples.duplicated(keep=False)]
 
-            if  len(df_samples) > 0:
+            if len(df_samples) > 0:
                 samples = df_samples.values.tolist()
 
         return samples
 
+    def _filter_failures(self, yi):
+        """Filter or replace failed objectives.
+
+        Args:
+            yi (list): a list of objectives.
+
+        Returns:
+            list: the filtered list.
+        """
+        if self.filter_failures in ["mean", "max"]:
+            yi_no_failure = [v for v in yi if v != OBJECTIVE_VALUE_FAILURE]
+
+            if self.filter_failures == "mean":
+                yi_failed_value = np.mean(yi_no_failure)
+            else:
+                yi_failed_value = np.max(yi_no_failure)
+
+            yi = [v if v != OBJECTIVE_VALUE_FAILURE else yi_failed_value for v in yi]
+
+        return yi
+
+    def _sample(self, X, y):
+
+        X = np.asarray(X, dtype="O")
+        y = np.asarray(y)
+        size = y.shape[0]
+
+        if self._sample_max_size > 0 and size > self._sample_max_size:
+            if self._sample_strategy == "quantile":
+                quantiles = np.quantile(y, [0.10, 0.25, 0.50, 0.75, 0.90])
+                int_size = self._sample_max_size // (len(quantiles) + 1)
+
+                Xs, ys = [], []
+                for i in range(len(quantiles) + 1):
+                    if i == 0:
+                        s = y < quantiles[i]
+                    elif i == len(quantiles):
+                        s = quantiles[i - 1] <= y
+                    else:
+                        s = (quantiles[i - 1] <= y) & (y < quantiles[i])
+
+                    idx = np.where(s)[0]
+                    idx = np.random.choice(idx, size=int_size, replace=True)
+                    Xi = X[idx]
+                    yi = y[idx]
+                    Xs.append(Xi)
+                    ys.append(yi)
+
+                X = np.concatenate(Xs, axis=0)
+                y = np.concatenate(ys, axis=0)
+
+        X = X.tolist()
+        y = y.tolist()
+        return X, y
+
+    def _ask_random_points(self, size=None):
+        samples = self.space.rvs(n_samples=self.n_points, random_state=self.rng)
+
+        samples = self._filter_duplicated(samples)
+
+        if size is None:
+            return samples[0]
+        else:
+            return samples[:size]
+
     def _ask(self):
         """Suggest next point at which to evaluate the objective.
 
@@ -524,12 +714,8 @@ def _ask(self):
         if self._n_initial_points > 0 or self.base_estimator_ is None:
             # this will not make a copy of `self.rng` and hence keep advancing
             # our random state.
-            if self._initial_samples is None:
-                samples = self.space.rvs(n_samples=self.n_points, random_state=self.rng)
-
-                samples = self._filter_duplicated(samples)
-
-                return samples[0]
+            if len(self._initial_samples) == 0:
+                return self._ask_random_points()
             else:
                 # The samples are evaluated starting form initial_samples[0]
                 return self._initial_samples[
@@ -544,8 +730,10 @@ def _ask(self):
 
             next_x = self._next_x
             if next_x is not None:
-                if not self.space.is_config_space and not self.space.is_ccs:
-                    min_delta_x = min([self.space.distance(next_x, xi) for xi in self.Xi])
+                if not self.space.is_config_space:
+                    min_delta_x = min(
+                        [self.space.distance(next_x, xi) for xi in self.Xi]
+                    )
                     if abs(min_delta_x) <= 1e-8:
                         warnings.warn(
                             "The objective has been evaluated " "at this point before."
@@ -583,8 +771,6 @@ def tell(self, x, y, fit=True):
         """
         if self.space.is_config_space:
             pass
-        elif self.space.is_ccs:
-            pass
         else:
             check_x_in_space(x, self.space)
 
@@ -638,96 +824,150 @@ def _tell(self, x, y, fit=True):
             transformed_bounds = np.array(self.space.transformed_bounds)
             est = clone(self.base_estimator_)
 
+            # handle failures
+            yi = self._filter_failures(self.yi)
+
+            # handle size of the sample fit to the estimator
+            Xi, yi = self._sample(self.Xi, yi)
+
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore")
-                Xtt = self.space.imp_const.fit_transform(self.space.transform(self.Xi))
-                est.fit(Xtt, self.yi)
-
-            if hasattr(self, "next_xs_") and self.acq_func == "gp_hedge":
-                self.gains_ -= est.predict(np.vstack(self.next_xs_))
+                Xtt = self.space.imp_const.fit_transform(self.space.transform(Xi))
+                est.fit(Xtt, yi)
 
-            if self.max_model_queue_size is None:
-                self.models.append(est)
-            elif len(self.models) < self.max_model_queue_size:
-                self.models.append(est)
+            # for qLCB save the fitted estimator and skip the selection
+            if self.acq_func == "qLCB":
+                self._est = est
             else:
-                # Maximum list size obtained, remove oldest model.
-                self.models.pop(0)
-                self.models.append(est)
-
-            # even with BFGS as optimizer we want to sample a large number
-            # of points and then pick the best ones as starting points
-            X_s = self.space.rvs(n_samples=self.n_points, random_state=self.rng)
-
-            X_s = self._filter_duplicated(X_s)
+                if hasattr(self, "next_xs_") and self.acq_func == "gp_hedge":
+                    self.gains_ -= est.predict(np.vstack(self.next_xs_))
+
+                if self.max_model_queue_size is None:
+                    self.models.append(est)
+                elif len(self.models) < self.max_model_queue_size:
+                    self.models.append(est)
+                else:
+                    # Maximum list size obtained, remove oldest model.
+                    self.models.pop(0)
+                    self.models.append(est)
+
+                # even with BFGS as optimizer we want to sample a large number
+                # of points and then pick the best ones as starting points
+                X_s = self.space.rvs(n_samples=self.n_points, random_state=self.rng)
+
+                X_s = self._filter_duplicated(X_s)
+
+                X = self.space.imp_const.fit_transform(self.space.transform(X_s))
+                self.next_xs_ = []
+                for cand_acq_func in self.cand_acq_funcs_:
+                    values = _gaussian_acquisition(
+                        X=X,
+                        model=est,
+                        y_opt=np.min(yi),
+                        acq_func=cand_acq_func,
+                        acq_func_kwargs=self.acq_func_kwargs,
+                    )
+
+                    # cache these values in case the strategy of ask is one-shot
+                    self._last_X = X
+                    self._last_values = values
+
+                    # Find the minimum of the acquisition function by randomly
+                    # sampling points from the space
+                    if self.acq_optimizer == "sampling":
+                        next_x = X[np.argmin(values)]
+
+                    elif self.acq_optimizer == "boltzmann_sampling":
+
+                        p = self.rng.uniform()
+                        if p <= self.boltzmann_psucc:
+                            next_x = X[np.argmin(values)]
+                        else:
+                            values = -values
+
+                            self._min_value = (
+                                self._min_value
+                                if self._min_value is None
+                                else min(values.min(), self._min_value)
+                            )
+                            self._max_value = (
+                                self._max_value
+                                if self._max_value is None
+                                else max(values.max(), self._max_value)
+                            )
 
-            X = self.space.imp_const.fit_transform(self.space.transform(X_s))
-            self.next_xs_ = []
-            for cand_acq_func in self.cand_acq_funcs_:
-                values = _gaussian_acquisition(
-                    X=X,
-                    model=est,
-                    y_opt=np.min(self.yi),
-                    acq_func=cand_acq_func,
-                    acq_func_kwargs=self.acq_func_kwargs,
-                )
-                # Find the minimum of the acquisition function by randomly
-                # sampling points from the space
-                if self.acq_optimizer == "sampling":
-                    next_x = X[np.argmin(values)]
-
-                # Use BFGS to find the mimimum of the acquisition function, the
-                # minimization starts from `n_restarts_optimizer` different
-                # points and the best minimum is used
-                elif self.acq_optimizer == "lbfgs":
-                    x0 = X[np.argsort(values)[: self.n_restarts_optimizer]]
-
-                    with warnings.catch_warnings():
-                        warnings.simplefilter("ignore")
-                        results = Parallel(n_jobs=self.n_jobs)(
-                            delayed(fmin_l_bfgs_b)(
-                                gaussian_acquisition_1D,
-                                x,
-                                args=(
-                                    est,
-                                    np.min(self.yi),
-                                    cand_acq_func,
-                                    self.acq_func_kwargs,
-                                ),
-                                bounds=self.space.transformed_bounds,
-                                approx_grad=False,
-                                maxiter=20,
+                            t = len(self.Xi)
+                            if t == 0:
+                                beta = 0
+                            else:
+                                beta = (
+                                    self.boltzmann_gamma
+                                    * np.log(t)
+                                    / np.abs(self._max_value - self._min_value)
+                                )
+
+                            probs = boltzman_distribution(values, beta)
+
+                            idx = np.argmax(self.rng.multinomial(1, probs))
+
+                            next_x = X[idx]
+
+                    # Use BFGS to find the mimimum of the acquisition function, the
+                    # minimization starts from `n_restarts_optimizer` different
+                    # points and the best minimum is used
+                    elif self.acq_optimizer == "lbfgs":
+                        x0 = X[np.argsort(values)[: self.n_restarts_optimizer]]
+
+                        with warnings.catch_warnings():
+                            warnings.simplefilter("ignore")
+                            results = Parallel(n_jobs=self.n_jobs)(
+                                delayed(fmin_l_bfgs_b)(
+                                    gaussian_acquisition_1D,
+                                    x,
+                                    args=(
+                                        est,
+                                        np.min(yi),
+                                        cand_acq_func,
+                                        self.acq_func_kwargs,
+                                    ),
+                                    bounds=self.space.transformed_bounds,
+                                    approx_grad=False,
+                                    maxiter=20,
+                                )
+                                for x in x0
                             )
-                            for x in x0
-                        )
 
-                    cand_xs = np.array([r[0] for r in results])
-                    cand_acqs = np.array([r[1] for r in results])
-                    next_x = cand_xs[np.argmin(cand_acqs)]
+                        cand_xs = np.array([r[0] for r in results])
+                        cand_acqs = np.array([r[1] for r in results])
+                        next_x = cand_xs[np.argmin(cand_acqs)]
+
+                    # lbfgs should handle this but just in case there are
+                    # precision errors.
+                    if not self.space.is_categorical:
+                        if not self.space.is_config_space:
+                            next_x = np.clip(
+                                next_x,
+                                transformed_bounds[:, 0],
+                                transformed_bounds[:, 1],
+                            )
+                    self.next_xs_.append(next_x)
 
-                # lbfgs should handle this but just in case there are
-                # precision errors.
-                if not self.space.is_categorical:
-                    if not self.space.is_config_space and not self.space.is_ccs:
-                        next_x = np.clip(
-                            next_x, transformed_bounds[:, 0], transformed_bounds[:, 1]
-                        )
-                self.next_xs_.append(next_x)
-
-            if self.acq_func == "gp_hedge":
-                logits = np.array(self.gains_)
-                logits -= np.max(logits)
-                exp_logits = np.exp(self.eta * logits)
-                probs = exp_logits / np.sum(exp_logits)
-                next_x = self.next_xs_[np.argmax(self.rng.multinomial(1, probs))]
-            else:
-                next_x = self.next_xs_[0]
+                if self.acq_func == "gp_hedge":
+                    logits = np.array(self.gains_)
+                    logits -= np.max(logits)
+                    exp_logits = np.exp(self.eta * logits)
+                    probs = exp_logits / np.sum(exp_logits)
+                    next_x = self.next_xs_[np.argmax(self.rng.multinomial(1, probs))]
+                else:
+                    next_x = self.next_xs_[0]
 
-            # note the need for [0] at the end
-            self._next_x = self.space.inverse_transform(next_x.reshape((1, -1)))[0]
+                # note the need for [0] at the end
+                self._next_x = self.space.inverse_transform(next_x.reshape((1, -1)))[0]
 
         # Pack results
-        result = create_result(self.Xi, self.yi, self.space, self.rng, models=self.models)
+        result = create_result(
+            self.Xi, self.yi, self.space, self.rng, models=self.models
+        )
 
         result.specs = self.specs
         return result
@@ -746,7 +986,10 @@ def _check_y_is_valid(self, x, y):
         # if y isn't a scalar it means we have been handed a batch of points
         elif is_listlike(y) and is_2Dlistlike(x):
             for y_value in y:
-                if not isinstance(y_value, Number):
+                if (
+                    not isinstance(y_value, Number)
+                    and y_value != OBJECTIVE_VALUE_FAILURE
+                ):
                     raise ValueError("expected y to be a list of scalars")
 
         elif is_listlike(x):
@@ -765,7 +1008,9 @@ def run(self, func, n_iter=1):
             x = self.ask()
             self.tell(x, func(x))
 
-        result = create_result(self.Xi, self.yi, self.space, self.rng, models=self.models)
+        result = create_result(
+            self.Xi, self.yi, self.space, self.rng, models=self.models
+        )
         result.specs = self.specs
         return result
 
@@ -789,6 +1034,8 @@ def get_result(self):
             OptimizeResult instance with the required information.
 
         """
-        result = create_result(self.Xi, self.yi, self.space, self.rng, models=self.models)
+        result = create_result(
+            self.Xi, self.yi, self.space, self.rng, models=self.models
+        )
         result.specs = self.specs
         return result
diff --git a/skopt/space/space.py b/skopt/space/space.py
index fbc5d108..8fb08fa1 100644
--- a/skopt/space/space.py
+++ b/skopt/space/space.py
@@ -1,6 +1,7 @@
 import numbers
 import numpy as np
 import yaml
+import sys
 
 from scipy.stats.distributions import randint
 from scipy.stats.distributions import rv_discrete
@@ -28,13 +29,7 @@
 
 
 import ConfigSpace as CS
-ccs_active = False
-try:
-    import cconfigspace as CCS
-    ccs_active = True
-except (ImportError, OSError) as a:
-    import warnings
-    warnings.warn("CCS could not be loaded and is deactivated: " + str(a), category=ImportWarning)
+from ConfigSpace.util import deactivate_inactive_hyperparameters
 
 from sklearn.impute import SimpleImputer
 
@@ -46,8 +41,7 @@ def __repr__(self):
 
 
 def _transpose_list_array(x):
-    """Transposes a list matrix
-    """
+    """Transposes a list matrix"""
 
     n_dims = len(x)
     assert n_dims > 0
@@ -140,9 +134,9 @@ def check_dimension(dimension, transform=None):
             "log-uniform",
         ]:
             return Integer(*dimension, transform=transform)
-        elif any([isinstance(dim, (float, int)) for dim in dimension[:2]]) and dimension[
-            2
-        ] in ["uniform", "log-uniform"]:
+        elif any(
+            [isinstance(dim, (float, int)) for dim in dimension[:2]]
+        ) and dimension[2] in ["uniform", "log-uniform"]:
             return Real(*dimension, transform=transform)
         else:
             return Categorical(dimension, transform=transform)
@@ -197,7 +191,7 @@ def transform(self, X):
 
     def inverse_transform(self, Xt):
         """Inverse transform samples from the warped space back into the
-           original space.
+        original space.
         """
         return self.transformer.inverse_transform(Xt)
 
@@ -242,11 +236,13 @@ def _uniform_inclusive(loc=0.0, scale=1.0):
     # XXX scale is very large.
     return uniform(loc=loc, scale=np.nextafter(scale, scale + 1.0))
 
+
 def _normal_inclusive(loc=0.0, scale=1.0, lower=-2, upper=2):
     assert lower <= upper
     a, b = (lower - loc) / scale, (upper - loc) / scale
     return truncnorm(a, b, loc=loc, scale=scale)
 
+
 class Real(Dimension):
     """Search space dimension that can take on any real value.
 
@@ -287,14 +283,29 @@ class Real(Dimension):
         can be float.
 
     """
-    def __init__(self, low, high, prior="uniform", base=10, transform=None,
-                 name=None, dtype=float, loc=None, scale=None):
+
+    def __init__(
+        self,
+        low,
+        high,
+        prior="uniform",
+        base=10,
+        transform=None,
+        name=None,
+        dtype=float,
+        loc=None,
+        scale=None,
+    ):
         if high <= low:
-            raise ValueError("the lower bound {} has to be less than the"
-                             " upper bound {}".format(low, high))
-        if prior not in ["uniform", "log-uniform"]:
-            raise ValueError("prior should be 'uniform' or 'log-uniform'"
-                             " got {}".format(prior))
+            raise ValueError(
+                "the lower bound {} has to be less than the"
+                " upper bound {}".format(low, high)
+            )
+        if prior not in ["uniform", "log-uniform", "normal"]:
+            raise ValueError(
+                "prior should be 'normal', 'uniform' or 'log-uniform'"
+                " got {}".format(prior)
+            )
         self.low = low
         self.high = high
         self.prior = prior
@@ -307,15 +318,23 @@ def __init__(self, low, high, prior="uniform", base=10, transform=None,
         self._rvs = None
         self.transformer = None
         self.transform_ = transform
-        if isinstance(self.dtype, str) and self.dtype\
-                not in ['float', 'float16', 'float32', 'float64']:
-            raise ValueError("dtype must be 'float', 'float16', 'float32'"
-                             "or 'float64'"
-                             " got {}".format(self.dtype))
-        elif isinstance(self.dtype, type) and \
-                not np.issubdtype(self.dtype, np.floating):
-            raise ValueError("dtype must be a np.floating subtype;"
-                             " got {}".format(self.dtype))
+        if isinstance(self.dtype, str) and self.dtype not in [
+            "float",
+            "float16",
+            "float32",
+            "float64",
+        ]:
+            raise ValueError(
+                "dtype must be 'float', 'float16', 'float32'"
+                "or 'float64'"
+                " got {}".format(self.dtype)
+            )
+        elif isinstance(self.dtype, type) and not np.issubdtype(
+            self.dtype, np.floating
+        ):
+            raise ValueError(
+                "dtype must be a np.floating subtype;" " got {}".format(self.dtype)
+            )
 
         if transform is None:
             transform = "identity"
@@ -347,7 +366,9 @@ def set_transformer(self, transform="identity"):
             self._rvs = _uniform_inclusive(0.0, 1.0)
             assert self.prior in ["uniform", "log-uniform"]
             if self.prior == "uniform":
-                self.transformer = Pipeline([Identity(), Normalize(self.low, self.high)])
+                self.transformer = Pipeline(
+                    [Identity(), Normalize(self.low, self.high)]
+                )
             else:
                 self.transformer = Pipeline(
                     [
@@ -391,7 +412,7 @@ def __repr__(self):
 
     def inverse_transform(self, Xt):
         """Inverse transform samples from the warped space back into the
-           original space.
+        original space.
         """
         inv_transform = super(Real, self).inverse_transform(Xt)
         if isinstance(inv_transform, list):
@@ -510,11 +531,14 @@ def __init__(
         scale=None,
     ):
         if high <= low:
-            raise ValueError("the lower bound {} has to be less than the"
-                             " upper bound {}".format(low, high))
+            raise ValueError(
+                "the lower bound {} has to be less than the"
+                " upper bound {}".format(low, high)
+            )
         if prior not in ["uniform", "log-uniform"]:
-            raise ValueError("prior should be 'uniform' or 'log-uniform'"
-                             " got {}".format(prior))
+            raise ValueError(
+                "prior should be 'uniform' or 'log-uniform'" " got {}".format(prior)
+            )
         self.low = low
         self.high = high
         self.prior = prior
@@ -631,7 +655,7 @@ def __repr__(self):
 
     def inverse_transform(self, Xt):
         """Inverse transform samples from the warped space back into the
-           original space.
+        original space.
         """
         # The concatenation of all transformed dimensions makes Xt to be
         # of type float, hence the required cast back to int.
@@ -673,7 +697,7 @@ def __contains__(self, point):
     @property
     def transformed_bounds(self):
         if self.transform_ == "normalize":
-            return 0., 1.
+            return 0.0, 1.0
         else:
             return (self.low, self.high)
 
@@ -768,8 +792,11 @@ def set_transformer(self, transform="onehot"):
             self.transformer.fit(self.categories)
         elif transform == "normalize":
             self.transformer = Pipeline(
-                [LabelEncoder(list(self.categories)),
-                 Normalize(0, len(self.categories) - 1, is_int=True)])
+                [
+                    LabelEncoder(list(self.categories)),
+                    Normalize(0, len(self.categories) - 1, is_int=True),
+                ]
+            )
         else:
             self.transformer = Identity()
             self.transformer.fit(self.categories)
@@ -801,7 +828,7 @@ def __repr__(self):
 
     def inverse_transform(self, Xt):
         """Inverse transform samples from the warped space back into the
-           original space.
+        original space.
         """
         # The concatenation of all transformed dimensions makes Xt to be
         # of type float, hence the required cast back to int.
@@ -845,7 +872,11 @@ def __contains__(self, point):
     @property
     def transformed_bounds(self):
         if self.transformed_size == 1:
-            return 0.0, 1.0
+            N = len(self.categories)
+            if self.transform_ == "label":
+                return 0.0, float(N - 1)
+            else:
+                return 0.0, 1.0
         else:
             return [(0.0, 1.0) for i in range(self.transformed_size)]
 
@@ -893,19 +924,23 @@ class Space(object):
             dimensions.
     """
 
-    def __init__(self, dimensions):
+    def __init__(self, dimensions, model_sdv=None):
+
+        # attributes used when a ConfigurationSpace from ConfigSpace is given
         self.is_config_space = False
-        self.is_ccs = False
         self.config_space_samples = None
-        self.ccs_samples = None
         self.config_space_explored = False
-        self.ccs_explored = False
+
         self.imp_const = SimpleImputer(
             missing_values=np.nan, strategy="constant", fill_value=-1000
         )
         self.imp_const_inv = SimpleImputer(
             missing_values=-1000, strategy="constant", fill_value=np.nan
         )
+
+        # attribute used when a generative model is used to sample
+        self.model_sdv = model_sdv
+
         self.hps_names = []
 
         if isinstance(dimensions, CS.ConfigurationSpace):
@@ -920,10 +955,20 @@ def __init__(self, dimensions):
             for x in hps:
                 self.hps_names.append(x.name)
                 if isinstance(x, CS.hyperparameters.CategoricalHyperparameter):
-                    vals = list(x.choices)
+                    categories = list(x.choices)
+                    if x.probabilities is not None:
+                        prior = list(x.probabilities)
+                    else:
+                        prior = [1.0/x.num_choices for _ in range(x.num_choices)]
                     if x.name in cond_hps:
-                        vals.append("NA")
-                    param = Categorical(vals, prior=x.probabilities, name=x.name)
+                        categories.append("NA")
+
+                        # remove p from prior
+                        p = 1 / len(categories)
+                        pi = p / (len(categories) - 1)
+                        prior = [prior_i - pi for prior_i in prior]
+                        prior.append(p)
+                    param = Categorical(categories, prior=prior, name=x.name)
                     space.append(param)
                     self.hps_type[x.name] = "Categorical"
                 elif isinstance(x, CS.hyperparameters.OrdinalHyperparameter):
@@ -959,64 +1004,22 @@ def __init__(self, dimensions):
                 elif isinstance(x, CS.hyperparameters.NormalFloatHyperparameter):
                     prior = "normal"
                     if x.log:
-                        raise ValueError("Unsupported 'log' transformation for NormalFloatHyperparameter.")
-                    param = Real(x.lower, x.upper, prior=prior, name=x.name,
-                                 loc=x.mu, scale=x.sigma)
+                        raise ValueError(
+                            "Unsupported 'log' transformation for NormalFloatHyperparameter."
+                        )
+                    param = Real(
+                        x.lower,
+                        x.upper,
+                        prior=prior,
+                        name=x.name,
+                        loc=x.mu,
+                        scale=x.sigma,
+                    )
                     space.append(param)
                     self.hps_type[x.name] = "Real"
                 else:
                     raise ValueError("Unknown Hyperparameter type.")
             dimensions = space
-        elif ccs_active and isinstance(dimensions, CCS.ConfigurationSpace):
-            self.is_ccs = True
-            self.ccs = dimensions
-            self.hps_type = {}
-
-            hps = self.ccs.hyperparameters
-            cond_hps = [x.name for x in self.ccs.conditional_hyperparameters]
-
-            space = []
-            for x in hps:
-                self.hps_names.append(x.name)
-                distrib = self.ccs.get_hyperparameter_distribution(x)[0]
-                if (isinstance(x, CCS.CategoricalHyperparameter) or
-                        isinstance(x, CCS.OrdinalHyperparameter) or
-                        isinstance(x, CCS.DiscreteHyperparameter)):
-                    vals = list(x.values)
-                    if x.name in cond_hps:
-                        vals.append("NA")
-                    if isinstance(distrib, CCS.RouletteDistribution):
-                        param = Categorical(vals, prior=distrib.areas, name=x.name)
-                    elif isinstance(distrib, CCS.UniformDistribution):
-                        param = Categorical(vals, name=x.name)
-                    else:
-                        raise ValueError("Unsupported distribution")
-                    space.append(param)
-                    self.hps_type[x.name] = "Categorical"
-                elif isinstance(x, CCS.NumericalHyperparameter):
-                    prior = "uniform"
-                    lower = x.lower
-                    upper = x.upper
-                    t = x.data_type
-                    if isinstance(distrib, CCS.UniformDistribution):
-                        if distrib.scale_type == CCS.ccs_scale_type.LOGARITHMIC:
-                            prior = "log-uniform"
-                    elif isinstance(distrib, CCS.NormalDistribution):
-                        prior = "normal"
-                        if distrib.scale_type == CCS.ccs_scale_type.LOGARITHMIC:
-                            raise ValueError("Unsupported 'log' transformation for CCS.NumericalHyperparameter with normal prior.")
-                    else:
-                        raise ValueError("Unsupported distribution")
-                    if CCS.ccs_numeric_type.NUM_INTEGER:
-                        param = Integer(lower, upper, prior=prior, name=x.name)
-                        self.hps_type[x.name] = "Integer"
-                    else:
-                        param = Real(lower, upper, prior=prior, name=x.name)
-                        self.hps_type[x.name] = "Real"
-                    space.append(param)
-                else:
-                    raise ValueError("Unknown Hyperparameter type")
-            dimensions = space
         self.dimensions = [check_dimension(dim) for dim in dimensions]
 
     def __eq__(self, other):
@@ -1089,7 +1092,11 @@ def from_yaml(cls, yml_path, namespace=None):
         with open(yml_path, "rb") as f:
             config = yaml.safe_load(f)
 
-        dimension_classes = {"real": Real, "integer": Integer, "categorical": Categorical}
+        dimension_classes = {
+            "real": Real,
+            "integer": Integer,
+            "categorical": Categorical,
+        }
 
         # Extract space options for configuration file
         if isinstance(config, dict):
@@ -1138,16 +1145,53 @@ def rvs(self, n_samples=1, random_state=None):
         points : list of lists, shape=(n_points, n_dims)
            Points sampled from the space.
         """
+
         rng = check_random_state(random_state)
         if self.is_config_space:
             req_points = []
 
-            confs = self.config_space.sample_configuration(n_samples)
-            if n_samples == 1:
-                confs = [confs]
-
             hps_names = self.config_space.get_hyperparameter_names()
-            for conf in confs:
+
+            if self.model_sdv is None:
+                confs = self.config_space.sample_configuration(n_samples)
+
+                if n_samples == 1:
+                    confs = [confs]
+            else:
+                confs = self.model_sdv.sample(n_samples)
+
+                sdv_names = confs.columns
+
+                new_hps_names = list(set(hps_names) - set(sdv_names))
+
+                # randomly sample the new hyperparameters
+                for name in new_hps_names:
+                    hp = self.config_space.get_hyperparameter(name)
+                    rvs = []
+                    for i in range(n_samples):
+                        v = hp._sample(rng)
+                        rv = hp._transform(v)
+                        rvs.append(rv)
+                    confs[name] = rvs
+
+                # reoder the column names
+                confs = confs[hps_names]
+
+                confs = confs.to_dict("records")
+                for idx, conf in enumerate(confs):
+                    cf = deactivate_inactive_hyperparameters(conf, self.config_space)
+                    confs[idx] = cf.get_dictionary()
+
+                    # TODO: remove because debug instructions
+                    # check if other conditions are not met; generate valid 1-exchange neighbor; need to test and develop the logic
+                    # print('conf invalid...generating valid 1-exchange neighbor')
+                    # neighborhood = get_one_exchange_neighbourhood(cf,1)
+                    # for new_config in neighborhood:
+                    #     print(new_config)
+                    #     print(new_config.is_valid_configuration())
+                    #     confs[idx] = new_config.get_dictionary()
+
+            for idx, conf in enumerate(confs):
                 point = []
                 for hps_name in hps_names:
                     val = np.nan
@@ -1159,33 +1203,27 @@ def rvs(self, n_samples=1, random_state=None):
                 req_points.append(point)
 
             return req_points
-        elif self.is_ccs:
-            confs = self.ccs.samples(n_samples)
-            hps = self.ccs.hyperparameters
-            points = []
-            for conf in confs:
-                point = []
-                values = conf.values
-                for i, hp in enumerate(hps):
-                    val = values[i]
-                    if CCS.ccs_inactive == val:
-                        if self.hps_type[hp.name] == "Categorical":
-                            val = "NA"
-                        else:
-                            val = np.nan
-                    point.append(val)
-                points.append(point)
-
-            return points
         else:
-            # Draw
-            columns = []
+            if self.model_sdv is None:
+                # Draw
+                columns = []
+                for dim in self.dimensions:
+                    columns.append(dim.rvs(n_samples=n_samples, random_state=rng))
+
+                # Transpose
+                return _transpose_list_array(columns)
+            else:
+                confs = self.model_sdv.sample(n_samples)  # sample from SDV
 
-            for dim in self.dimensions:
-                columns.append(dim.rvs(n_samples=n_samples, random_state=rng))
+                columns = []
+                for dim in self.dimensions:
+                    if dim.name in confs.columns:
+                        columns.append(confs[dim.name].values.tolist())
+                    else:
+                        columns.append(dim.rvs(n_samples=n_samples, random_state=rng))
 
-            # Transpose
-            return _transpose_list_array(columns)
+                # Transpose
+                return _transpose_list_array(columns)
 
     def set_transformer(self, transform):
         """Sets the transformer of all dimension objects to `transform`
@@ -1256,7 +1294,7 @@ def transform(self, X):
         # Repack as an array
         Xt = np.hstack([np.asarray(c).reshape((len(X), -1)) for c in columns])
 
-        if False and (self.is_config_space or self.is_ccs):
+        if False and self.is_config_space:
             self.imp_const.fit(Xt)
             Xtt = self.imp_const.transform(Xt)
             Xt = Xtt
@@ -1374,8 +1412,10 @@ def _get(dimension_name):
             # Note that we do not check whether the names are really strings.
             dims = [_get(dimension_name=name) for name in dimension_names]
         else:
-            msg = "Dimension name should be either string or" \
-                  "list of strings, but got {}."
+            msg = (
+                "Dimension name should be either string or"
+                "list of strings, but got {}."
+            )
             raise ValueError(msg.format(type(dimension_names)))
 
         return dims

From 18d0d65a16e967a47eda5f2bf2ce67ed644bcfd2 Mon Sep 17 00:00:00 2001
From: Thomas L Randall <tlranda@clemson.edu>
Date: Wed, 22 Jun 2022 15:14:41 -0500
Subject: [PATCH 2/5] Return CCS features that were removed in Deephyper
 version

---
 skopt/space/space.py | 78 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 77 insertions(+), 1 deletion(-)

diff --git a/skopt/space/space.py b/skopt/space/space.py
index 8fb08fa1..37be8961 100644
--- a/skopt/space/space.py
+++ b/skopt/space/space.py
@@ -29,6 +29,13 @@
 
 
 import ConfigSpace as CS
+ccs_active = False
+try:
+    import cconfigspace as CCS
+    ccs_active = True
+except (ImportError, OSError) as a:
+    import warnings
+    warnings.warn(f"CCS could not be loaded and is deactivated: {a}", category=ImportWarning)
 from ConfigSpace.util import deactivate_inactive_hyperparameters
 
 from sklearn.impute import SimpleImputer
@@ -928,8 +935,11 @@ def __init__(self, dimensions, model_sdv=None):
 
         # attributes used when a ConfigurationSpace from ConfigSpace is given
         self.is_config_space = False
+        self.is_ccs = False
         self.config_space_samples = None
+        self.ccs_samples = None
         self.config_space_explored = False
+        self.ccs_explored = False
 
         self.imp_const = SimpleImputer(
             missing_values=np.nan, strategy="constant", fill_value=-1000
@@ -1020,6 +1030,56 @@ def __init__(self, dimensions, model_sdv=None):
                 else:
                     raise ValueError("Unknown Hyperparameter type.")
             dimensions = space
+        elif ccs_active and isinstance(dimensions, CCS.ConfigurationSpace):
+            self.is_ccs = True
+            self.ccs = dimensions
+            self.hps_type = {}
+
+            hps = self.ccs.hyperparameters
+            cond_hps = [x.name for x in self.ccs.conditional_hyperparameters]
+
+            space = []
+            for x in hps:
+                self.hps_names.append(x.name)
+                distrib = self.ccs.get_hyperparameter_distribution(x)[0]
+                if (isinstance(x, CCS.CategoricalHyperparameter) or
+                        isinstance(x, CCS.OrdinalHyperparameter) or
+                        isinstance(x, CCS.DiscreteHyperparameter)):
+                    vals = list(x.values)
+                    if x.name in cond_hps:
+                        vals.append("NA")
+                    if isinstance(distrib, CCS.RouletteDistribution):
+                        param = Categorical(vals, prior=distrib.areas, name=x.name)
+                    elif isinstance(distrib, CCS.UniformDistribution):
+                        param = Categorical(vals, name=x.name)
+                    else:
+                        raise ValueError("Unsupported distribution")
+                    space.append(param)
+                    self.hps_type[x.name] = "Categorical"
+                elif isinstance(x, CCS.NumericalHyperparameter):
+                    prior = "uniform"
+                    lower = x.lower
+                    upper = x.upper
+                    t = x.data_type
+                    if isinstance(distrib, CCS.UniformDistribution):
+                        if distrib.scale_type == CCS.ccs_scale_type.LOGARITHMIC:
+                            prior = "log-uniform"
+                    elif isinstance(distrib, CCS.NormalDistribution):
+                        prior = "normal"
+                        if distrib.scale_type == CCS.ccs_scale_type.LOGARITHMIC:
+                            raise ValueError("Unsupported 'log' transformation for CCS.NumericalHyperparameter with normal prior.")
+                    else:
+                        raise ValueError("Unsupported distribution")
+                    if CCS.ccs_numeric_type.NUM_INTEGER:
+                        param = Integer(lower, upper, prior=prior, name=x.name)
+                        self.hps_type[x.name] = "Integer"
+                    else:
+                        param = Real(lower, upper, prior=prior, name=x.name)
+                        self.hps_type[x.name] = "Real"
+                    space.append(param)
+                    else:
+                        raise ValueError("Unknown Hyperparameter type")
+                dimensions = space
         self.dimensions = [check_dimension(dim) for dim in dimensions]
 
     def __eq__(self, other):
@@ -1203,6 +1263,22 @@ def rvs(self, n_samples=1, random_state=None):
                 req_points.append(point)
 
             return req_points
+        elif self.is_ccs:
+            confs = self.ccs.samples(n_samples)
+            hps = self.ccs.hyperparameters
+            points = []
+            for conf in confs:
+                point = []
+                for hp in hps:
+                    val = conf.value(hp)
+                    if CCS.ccs_inactive == val:
+                        if self.hps_type[hp.name] == "Categorical":
+                            val = "NA"
+                        else:
+                            val = np.nan
+                    point.append(val)
+                points.append(point)
+            return points
         else:
             if self.model_sdv is None:
                 # Draw
@@ -1294,7 +1370,7 @@ def transform(self, X):
         # Repack as an array
         Xt = np.hstack([np.asarray(c).reshape((len(X), -1)) for c in columns])
 
-        if False and self.is_config_space:
+        if False and (self.is_config_space or self.is_ccs):
             self.imp_const.fit(Xt)
             Xtt = self.imp_const.transform(Xt)
             Xt = Xtt

From d5f6160a04b212730616f04e9c4d4f4e23125ed6 Mon Sep 17 00:00:00 2001
From: Thomas L Randall <tlranda@clemson.edu>
Date: Wed, 22 Jun 2022 15:23:44 -0500
Subject: [PATCH 3/5] Semantic indentation that did not match prior version

---
 skopt/space/space.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/skopt/space/space.py b/skopt/space/space.py
index 37be8961..25619161 100644
--- a/skopt/space/space.py
+++ b/skopt/space/space.py
@@ -1077,9 +1077,9 @@ def __init__(self, dimensions, model_sdv=None):
                         param = Real(lower, upper, prior=prior, name=x.name)
                         self.hps_type[x.name] = "Real"
                     space.append(param)
-                    else:
-                        raise ValueError("Unknown Hyperparameter type")
-                dimensions = space
+                else:
+                    raise ValueError("Unknown Hyperparameter type")
+            dimensions = space
         self.dimensions = [check_dimension(dim) for dim in dimensions]
 
     def __eq__(self, other):
@@ -1285,9 +1285,6 @@ def rvs(self, n_samples=1, random_state=None):
                 columns = []
                 for dim in self.dimensions:
                     columns.append(dim.rvs(n_samples=n_samples, random_state=rng))
-
-                # Transpose
-                return _transpose_list_array(columns)
             else:
                 confs = self.model_sdv.sample(n_samples)  # sample from SDV
 
@@ -1297,9 +1294,8 @@ def rvs(self, n_samples=1, random_state=None):
                         columns.append(confs[dim.name].values.tolist())
                     else:
                         columns.append(dim.rvs(n_samples=n_samples, random_state=rng))
-
-                # Transpose
-                return _transpose_list_array(columns)
+            # Transpose
+            return _transpose_list_array(columns)
 
     def set_transformer(self, transform):
         """Sets the transformer of all dimension objects to `transform`

From a8c1c61f8573ebfd9b285ba56f681d4722b9c16c Mon Sep 17 00:00:00 2001
From: Thomas L Randall <tlranda@clemson.edu>
Date: Wed, 22 Jun 2022 16:32:35 -0500
Subject: [PATCH 4/5] Fix semantic indent error

---
 skopt/space/space.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/skopt/space/space.py b/skopt/space/space.py
index 25619161..8b9a7a7c 100644
--- a/skopt/space/space.py
+++ b/skopt/space/space.py
@@ -1079,7 +1079,7 @@ def __init__(self, dimensions, model_sdv=None):
                     space.append(param)
                 else:
                     raise ValueError("Unknown Hyperparameter type")
-            dimensions = space
+                dimensions = space
         self.dimensions = [check_dimension(dim) for dim in dimensions]
 
     def __eq__(self, other):
@@ -1285,6 +1285,9 @@ def rvs(self, n_samples=1, random_state=None):
                 columns = []
                 for dim in self.dimensions:
                     columns.append(dim.rvs(n_samples=n_samples, random_state=rng))
+
+                # Transpose
+                return _transpose_list_array(columns)
             else:
                 confs = self.model_sdv.sample(n_samples)  # sample from SDV
 
@@ -1294,8 +1297,9 @@ def rvs(self, n_samples=1, random_state=None):
                         columns.append(confs[dim.name].values.tolist())
                     else:
                         columns.append(dim.rvs(n_samples=n_samples, random_state=rng))
-            # Transpose
-            return _transpose_list_array(columns)
+
+                # Transpose
+                return _transpose_list_array(columns)
 
     def set_transformer(self, transform):
         """Sets the transformer of all dimension objects to `transform`

From fc851063cf3c23cd7b564fcff503822fa02a4fc7 Mon Sep 17 00:00:00 2001
From: Thomas L Randall <tlranda@clemson.edu>
Date: Wed, 10 Aug 2022 12:33:26 -0500
Subject: [PATCH 5/5] Merge bugfixes from local

Prevent skopt optimizers from selecting rows they already have selected
Attempt to prevent unconstrained models from returning zero rows when generation fails to be 'close'
---
 skopt/optimizer/optimizer.py | 10 +++---
 skopt/space/space.py         | 62 +++++++++++++++++++++++++++++++++++-
 2 files changed, 66 insertions(+), 6 deletions(-)

diff --git a/skopt/optimizer/optimizer.py b/skopt/optimizer/optimizer.py
index a8767260..ea884da6 100644
--- a/skopt/optimizer/optimizer.py
+++ b/skopt/optimizer/optimizer.py
@@ -30,7 +30,6 @@
 from ..utils import normalize_dimensions
 from ..utils import cook_initial_point_generator
 
-
 class ExhaustedSearchSpace(RuntimeError):
     """ "Raised when the search cannot sample new points from the ConfigSpace."""
 
@@ -180,7 +179,7 @@ class Optimizer(object):
         Regression models used to fit observations and compute acquisition
         function.
     space : Space
-        An instance of :class:`skopt.space.Space`. Stores parameter search
+        An instance of :class:`deephyper.skopt.space.Space`. Stores parameter search
         space used to sample points, bounds, and type of parameters.
 
     """
@@ -327,9 +326,7 @@ def __init__(
 
         # keep track of the generative model from sdv
         self.model_sdv = model_sdv
-
         self.space = Space(dimensions, model_sdv=self.model_sdv)
-
         self._initial_samples = [] if initial_points is None else initial_points[:]
         self._initial_point_generator = cook_initial_point_generator(
             initial_point_generator
@@ -626,7 +623,10 @@ def _filter_duplicated(self, samples):
                 hps_names = self.space.dimension_names
 
             df_samples = pd.DataFrame(data=samples, columns=hps_names, dtype="O")
-            df_samples = df_samples[~df_samples.duplicated(keep="first")]
+            # Also drop things already seen in Xi
+            xi_samples = pd.DataFrame(data=self.Xi, columns=hps_names, dtype="O")
+            # NO duplicates should be present from this point on
+            df_samples = pd.concat((df_samples, xi_samples)).drop_duplicates(keep=False)
 
             if len(self.sampled) > 0:
                 df_history = pd.DataFrame(data=self.sampled, columns=hps_names)
diff --git a/skopt/space/space.py b/skopt/space/space.py
index 8b9a7a7c..303558d9 100644
--- a/skopt/space/space.py
+++ b/skopt/space/space.py
@@ -1,5 +1,6 @@
 import numbers
 import numpy as np
+import pandas as pd
 import yaml
 import sys
 
@@ -1185,6 +1186,62 @@ def from_yaml(cls, yml_path, namespace=None):
 
         return space
 
+    def sdv_close_enough(self, samples, rows, col, target):
+        out = []
+        criterion = self.config_space.approximate_conditions_dict['criterion']
+        # Eliminate unfit rows (beyond nearest criterion in wrong direction)
+        if target > criterion[-1]:
+            possible = samples[samples[col] > criterion[-1]]
+        elif target < criterion[0]:
+            possible = samples[samples[col] < criterion[0]]
+        else:
+            # Find target criterion in middle using detection of sign change of difference
+            sign_idx = list(np.sign(pd.Series(criterion)-target).diff()[1:].ne(0)).index(True)
+            lower, upper = criterion[sign_idx:sign_idx+2]
+            possible = samples[(samples[col] > lower) & (samples[col] < upper)]
+        # In case absolutely no rows are fit, we'll revert to choosing from the best available
+        if len(possible) == 0:
+            possible = samples
+        # Prioritize closest rows first
+        dists = (possible[col]-target).abs().sort_values().index[:rows]
+        return possible.loc[dists].reset_index(drop=True)
+
+    def sdv_sample_approximate_conditions(self, n_samples):
+        approximate_conditions_dict = self.config_space.approximate_conditions_dict
+        if not hasattr(self, 'has_support'):
+            import inspect
+            try:
+                source = inspect.getsource(self.model_sdv._sample)
+                self.has_support = not("raise NotImplementedError" in source\
+                              and "doesn't support conditional sampling" in source)
+            except AttributeError:
+                self.has_support = False
+        if self.has_support:
+            return self.model_sdv.sample_conditions(approximate_conditions_dict['conditions'])
+        # Else use manual approximation for sampling bias
+        selected = []
+        prev_len = -1
+        cur_len = 0
+        params = approximate_conditions_dict['param_names']
+        # Lack of change may be indication that no other rows can be found
+        while prev_len < n_samples and cur_len != prev_len:
+            prev_len = cur_len
+            samples = self.model_sdv.sample(num_rows=n_samples, randomize_samples=False)
+            candidate = []
+            for cond in approximate_conditions_dict['conditions']:
+                n_rows = cond.get_num_rows()
+                for (col, target) in cond.get_column_values().items():
+                    candidate.append(self.sdv_close_enough(samples, n_rows, col, target))
+            candidate = pd.concat(candidate).drop_duplicates(subset=params)
+            selected.append(candidate)
+            cur_len = sum(map(len, selected))
+        selected = pd.concat(selected).drop_duplicates(subset=params)
+        # FORCE conditions to be held in the data
+        for cond in approximate_conditions_dict['conditions']:
+            for (col, target) in cond.get_column_values().items():
+                selected[col] = target
+        return selected
+
     def rvs(self, n_samples=1, random_state=None):
         """Draw random samples.
 
@@ -1218,7 +1275,10 @@ def rvs(self, n_samples=1, random_state=None):
                 if n_samples == 1:
                     confs = [confs]
             else:
-                confs = self.model_sdv.sample(n_samples)
+                if hasattr(self.config_space, 'approximate_conditions_dict'):
+                    confs = self.sdv_sample_approximate_conditions(n_samples)
+                else:
+                    confs = self.model_sdv.sample(n_samples)
 
                 sdv_names = confs.columns