From 0f0823a4917fead39f1be4e17b862ea202877e0f Mon Sep 17 00:00:00 2001 From: shuki Date: Tue, 12 Feb 2019 19:31:22 +0100 Subject: [PATCH 1/4] FIX issue 98 by ignoring forbidden clauses --- pimp/evaluator/local_parameter_importance.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pimp/evaluator/local_parameter_importance.py b/pimp/evaluator/local_parameter_importance.py index fc28f0f..f8edcc0 100644 --- a/pimp/evaluator/local_parameter_importance.py +++ b/pimp/evaluator/local_parameter_importance.py @@ -178,8 +178,17 @@ def run(self) -> OrderedDict: overall_var = {} overall_imp = {} all_preds = [] - def_perf, def_var = self._predict_over_instance_set(impute_inactive_values(self.cs.get_default_configuration())) - inc_perf, inc_var = self._predict_over_instance_set(impute_inactive_values(self.incumbent)) + + # Create ConfigSpace object without forbidden clauses to use impute_active_values-method + # This can be done because we don't actually use the Configuration-object anywhere + configspace_no_forbidden = deepcopy(self.incumbent.configuration_space) + configspace_no_forbidden.forbidden_clauses = [] + default_no_forbidden, incumbent_no_forbidden = self.cs.get_default_configuration(), deepcopy(self.incumbent) + default_no_forbidden.configuration_space = configspace_no_forbidden + incumbent_no_forbidden.configuration_space = configspace_no_forbidden + + def_perf, def_var = self._predict_over_instance_set(impute_inactive_values(default_no_forbidden)) + inc_perf, inc_var = self._predict_over_instance_set(impute_inactive_values(incumbent_no_forbidden)) delta = def_perf - inc_perf pbar = tqdm(range(self._sampled_neighbors), ascii=True, disable=not self.verbose) sum_var = 0 @@ -207,7 +216,7 @@ def run(self) -> OrderedDict: new_array = incumbent_array.copy() new_array = change_hp_value(self.incumbent.configuration_space, new_array, param, unit_neighbor, index) - new_configuration = impute_inactive_values(Configuration(self.incumbent.configuration_space, + new_configuration = impute_inactive_values(Configuration(configspace_no_forbidden, vector=new_array)) mean, var = self._predict_over_instance_set(new_configuration) performance_dict[param].append(mean) @@ -221,7 +230,7 @@ def run(self) -> OrderedDict: neighborhood_dict[param][0] = np.array(incumbent_array[index]) neighborhood_dict[param][1] = [self.incumbent[param]] if not added_inc: - mean, var = self._predict_over_instance_set(impute_inactive_values(self.incumbent)) + mean, var = self._predict_over_instance_set(impute_inactive_values(incumbent_no_forbidden)) performance_dict[param].append(mean) overall_var[param].append(mean) variance_dict[param].append(var) From ba2010fda248237ea2f3b2375fe5762babc5b5c6 Mon Sep 17 00:00:00 2001 From: shuki Date: Wed, 13 Feb 2019 15:11:20 +0100 Subject: [PATCH 2/4] FIX forbidden clauses #98 for fanova --- pimp/evaluator/fanova.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pimp/evaluator/fanova.py b/pimp/evaluator/fanova.py index eb71ddb..06c8a34 100644 --- a/pimp/evaluator/fanova.py +++ b/pimp/evaluator/fanova.py @@ -1,4 +1,5 @@ from collections import OrderedDict +from copy import deepcopy import pickle import warnings @@ -52,6 +53,11 @@ def __init__(self, scenario, cs, model, to_evaluate: int, runhist: RunHistory, r self.cs.add_hyperparameters(new_hyperparameters) self.cs_contained_constant = True + # Ignore forbidden clauses to allow imputing inactive values for epm + self.cs_no_forbidden = deepcopy(self.cs) + self.cs_no_forbidden.forbidden_clauses = [] + default_no_forbidden = impute_inactive_values(self.cs_no_forbidden.get_default_configuration()) + # This way the instance features in X are ignored and a new forest is constructed if self.model.instance_features is None: self.logger.info('No preprocessing necessary') @@ -65,11 +71,11 @@ def __init__(self, scenario, cs, model, to_evaluate: int, runhist: RunHistory, r cutoffs = (-np.inf, np.inf) if minimize: cutoffs = (-np.inf, self.model.predict_marginalized_over_instances( - np.array([impute_inactive_values(self.cs.get_default_configuration()).get_array()]))[0].flatten()[0] + np.array([default_no_forbidden.get_array()]))[0].flatten()[0] ) elif minimize is False: cutoffs = (self.model.predict_marginalized_over_instances( - np.array([impute_inactive_values( self.cs.get_default_configuration()).get_array()]))[0].flatten()[0], + np.array([default_no_forbidden.get_array()]))[0].flatten()[0], np.inf) self.evaluator = fanova_pyrfr(X=self.X, Y=self.y.flatten(), config_space=self.cs, seed=self.rng.randint(2**31-1), cutoffs=cutoffs) @@ -91,6 +97,8 @@ def _preprocess(self, runhistory): configs = runhistory.get_all_configs() if self.cs_contained_constant: configs = [Configuration(self.cs, vector=c.get_array()) for c in configs] + # Remove forbidden-constraints + configs = [Configuration(self.cs_no_forbidden, vector=c.get_array()) for c in configs] X_non_hyper, X_prime = [], [] for config in configs: config = impute_inactive_values(config).get_array() From 2857ef452b4bf21c3c657e6d6fb0d0cd84c85cba Mon Sep 17 00:00:00 2001 From: shuki Date: Wed, 13 Feb 2019 15:55:32 +0100 Subject: [PATCH 3/4] REVERT changes, ADD new strategy of try-except ForbiddenValueError --- pimp/evaluator/fanova.py | 21 ++++++++-------- pimp/evaluator/local_parameter_importance.py | 25 +++++++++----------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/pimp/evaluator/fanova.py b/pimp/evaluator/fanova.py index 06c8a34..42ec23b 100644 --- a/pimp/evaluator/fanova.py +++ b/pimp/evaluator/fanova.py @@ -1,5 +1,4 @@ from collections import OrderedDict -from copy import deepcopy import pickle import warnings @@ -15,6 +14,7 @@ from ConfigSpace.configuration_space import ConfigurationSpace, Configuration from ConfigSpace.util import impute_inactive_values from ConfigSpace.hyperparameters import CategoricalHyperparameter, Constant +from ConfigSpace.exceptions import ForbiddenValueError try: from fanova import fANOVA as fanova_pyrfr @@ -53,11 +53,6 @@ def __init__(self, scenario, cs, model, to_evaluate: int, runhist: RunHistory, r self.cs.add_hyperparameters(new_hyperparameters) self.cs_contained_constant = True - # Ignore forbidden clauses to allow imputing inactive values for epm - self.cs_no_forbidden = deepcopy(self.cs) - self.cs_no_forbidden.forbidden_clauses = [] - default_no_forbidden = impute_inactive_values(self.cs_no_forbidden.get_default_configuration()) - # This way the instance features in X are ignored and a new forest is constructed if self.model.instance_features is None: self.logger.info('No preprocessing necessary') @@ -71,11 +66,11 @@ def __init__(self, scenario, cs, model, to_evaluate: int, runhist: RunHistory, r cutoffs = (-np.inf, np.inf) if minimize: cutoffs = (-np.inf, self.model.predict_marginalized_over_instances( - np.array([default_no_forbidden.get_array()]))[0].flatten()[0] + np.array([impute_inactive_values(self.cs.get_default_configuration()).get_array()]))[0].flatten()[0] ) elif minimize is False: cutoffs = (self.model.predict_marginalized_over_instances( - np.array([default_no_forbidden.get_array()]))[0].flatten()[0], + np.array([impute_inactive_values( self.cs.get_default_configuration()).get_array()]))[0].flatten()[0], np.inf) self.evaluator = fanova_pyrfr(X=self.X, Y=self.y.flatten(), config_space=self.cs, seed=self.rng.randint(2**31-1), cutoffs=cutoffs) @@ -97,17 +92,21 @@ def _preprocess(self, runhistory): configs = runhistory.get_all_configs() if self.cs_contained_constant: configs = [Configuration(self.cs, vector=c.get_array()) for c in configs] - # Remove forbidden-constraints - configs = [Configuration(self.cs_no_forbidden, vector=c.get_array()) for c in configs] X_non_hyper, X_prime = [], [] + skipped_forbidden = 0 for config in configs: - config = impute_inactive_values(config).get_array() + try: + config = impute_inactive_values(config).get_array() + except ForbiddenValueError: + skipped_forbidden += 1 + continue X_prime.append(config) X_non_hyper.append(config) for idx, param in enumerate(self.cs.get_hyperparameters()): if not (isinstance(param, CategoricalHyperparameter) or isinstance(param, Constant)): X_non_hyper[-1][idx] = param._transform(X_non_hyper[-1][idx]) + self.logger.debug("Skipped %d forbidden configurations", skipped_forbidden) X_non_hyper = np.array(X_non_hyper) X_prime = np.array(X_prime) y_prime = np.array(self.model.predict_marginalized_over_instances(X_prime)[0]) diff --git a/pimp/evaluator/local_parameter_importance.py b/pimp/evaluator/local_parameter_importance.py index f8edcc0..f6f5fbb 100644 --- a/pimp/evaluator/local_parameter_importance.py +++ b/pimp/evaluator/local_parameter_importance.py @@ -178,17 +178,8 @@ def run(self) -> OrderedDict: overall_var = {} overall_imp = {} all_preds = [] - - # Create ConfigSpace object without forbidden clauses to use impute_active_values-method - # This can be done because we don't actually use the Configuration-object anywhere - configspace_no_forbidden = deepcopy(self.incumbent.configuration_space) - configspace_no_forbidden.forbidden_clauses = [] - default_no_forbidden, incumbent_no_forbidden = self.cs.get_default_configuration(), deepcopy(self.incumbent) - default_no_forbidden.configuration_space = configspace_no_forbidden - incumbent_no_forbidden.configuration_space = configspace_no_forbidden - - def_perf, def_var = self._predict_over_instance_set(impute_inactive_values(default_no_forbidden)) - inc_perf, inc_var = self._predict_over_instance_set(impute_inactive_values(incumbent_no_forbidden)) + def_perf, def_var = self._predict_over_instance_set(impute_inactive_values(self.cs.get_default_configuration())) + inc_perf, inc_var = self._predict_over_instance_set(impute_inactive_values(self.incumbent)) delta = def_perf - inc_perf pbar = tqdm(range(self._sampled_neighbors), ascii=True, disable=not self.verbose) sum_var = 0 @@ -202,6 +193,7 @@ def run(self) -> OrderedDict: added_inc = False inc_at = 0 # Iterate over neighbors + skipped_forbidden_neighbors = 0 for unit_neighbor, neighbor in zip(neighborhood_dict[param][0], neighborhood_dict[param][1]): if not added_inc: if unit_neighbor > incumbent_array[index]: @@ -216,13 +208,18 @@ def run(self) -> OrderedDict: new_array = incumbent_array.copy() new_array = change_hp_value(self.incumbent.configuration_space, new_array, param, unit_neighbor, index) - new_configuration = impute_inactive_values(Configuration(configspace_no_forbidden, - vector=new_array)) + try: + new_configuration = impute_inactive_values(Configuration(self.incumbent.configuration_space, + vector=new_array)) + except ForbiddenValueError: + skipped_forbidden_neighbors += 1 + continue mean, var = self._predict_over_instance_set(new_configuration) performance_dict[param].append(mean) overall_var[param].append(mean) variance_dict[param].append(var) pbar.update(1) + self.logger.debug("Skipped %d forbidden neighbors", skipped_forbidden_neighbors) if len(neighborhood_dict[param][0]) > 0: neighborhood_dict[param][0] = np.insert(neighborhood_dict[param][0], inc_at, incumbent_array[index]) neighborhood_dict[param][1] = np.insert(neighborhood_dict[param][1], inc_at, self.incumbent[param]) @@ -230,7 +227,7 @@ def run(self) -> OrderedDict: neighborhood_dict[param][0] = np.array(incumbent_array[index]) neighborhood_dict[param][1] = [self.incumbent[param]] if not added_inc: - mean, var = self._predict_over_instance_set(impute_inactive_values(incumbent_no_forbidden)) + mean, var = self._predict_over_instance_set(impute_inactive_values(incumbent)) performance_dict[param].append(mean) overall_var[param].append(mean) variance_dict[param].append(var) From b6976f6e7313187b6333105ba48ed834ea7df6a7 Mon Sep 17 00:00:00 2001 From: shuki Date: Sun, 24 Feb 2019 15:21:00 +0100 Subject: [PATCH 4/4] FIX incumbent not defined error --- pimp/evaluator/local_parameter_importance.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pimp/evaluator/local_parameter_importance.py b/pimp/evaluator/local_parameter_importance.py index f6f5fbb..60a15d3 100644 --- a/pimp/evaluator/local_parameter_importance.py +++ b/pimp/evaluator/local_parameter_importance.py @@ -219,7 +219,11 @@ def run(self) -> OrderedDict: overall_var[param].append(mean) variance_dict[param].append(var) pbar.update(1) - self.logger.debug("Skipped %d forbidden neighbors", skipped_forbidden_neighbors) + self.logger.debug("Skipped %d (of %d) forbidden neighbors", skipped_forbidden_neighbors, + len(neighborhood_dict[param][0])) + if skipped_forbidden_neighbors == len(neighborhood_dict[param][0]): + self.logger.debug("No valid neighbors found for %s, skipping.", param) + continue if len(neighborhood_dict[param][0]) > 0: neighborhood_dict[param][0] = np.insert(neighborhood_dict[param][0], inc_at, incumbent_array[index]) neighborhood_dict[param][1] = np.insert(neighborhood_dict[param][1], inc_at, self.incumbent[param]) @@ -227,7 +231,7 @@ def run(self) -> OrderedDict: neighborhood_dict[param][0] = np.array(incumbent_array[index]) neighborhood_dict[param][1] = [self.incumbent[param]] if not added_inc: - mean, var = self._predict_over_instance_set(impute_inactive_values(incumbent)) + mean, var = self._predict_over_instance_set(impute_inactive_values(self.incumbent)) performance_dict[param].append(mean) overall_var[param].append(mean) variance_dict[param].append(var)