From b30f7a57a9d7ed56c9a29b2250769e7c5bd849e8 Mon Sep 17 00:00:00 2001 From: Koen van der Blom Date: Thu, 28 Jan 2021 14:55:50 +0100 Subject: [PATCH 1/3] Fix crashes resulting from divisions by zero by incorrect treatment of timeouts --- autofolio/validation/validate.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/autofolio/validation/validate.py b/autofolio/validation/validate.py index 5a0968f..c8fcc0a 100644 --- a/autofolio/validation/validate.py +++ b/autofolio/validation/validate.py @@ -1,4 +1,5 @@ import logging +import sys from aslib_scenario.aslib_scenario import ASlibScenario @@ -67,8 +68,15 @@ def show(self, remove_unsolvable: bool=True): if self.runtime_cutoff: n_samples = timeouts + self.solved - self.logger.info("PAR1: %.4f" % (par1 / n_samples)) - self.logger.info("PAR10: %.4f" % (par10 / n_samples)) + self.logger.debug("n_samples = timeouts + self.solved: %d" % n_samples) + if n_samples == 0: + par1_out = sys.maxsize + par10_out = sys.maxsize + else: + par1_out = par1 / n_samples + par10_out = par10 / n_samples + self.logger.info("PAR1: %.4f" % (par1_out)) + self.logger.info("PAR10: %.4f" % (par10_out)) self.logger.info("Timeouts: %d / %d" % (timeouts, n_samples)) self.logger.info("Presolved during feature computation: %d / %d" % (self.presolved_feats, n_samples)) self.logger.info("Solved: %d / %d" % (self.solved, n_samples)) @@ -79,17 +87,26 @@ def show(self, remove_unsolvable: bool=True): self.logger.info("Number of instances: %d" %(n_samples)) self.logger.info("Average Solution Quality: %.4f" % (par1 / n_samples)) par10 = par1 - - self.logger.info("Oracle: %.4f" %(oracle / n_samples)) + + if n_samples == 0: + oracle_out = sys.maxsize + else: + oracle_out = oracle / n_samples + self.logger.info("Oracle: %.4f" %(oracle_out)) if sbs > 0: self.logger.info("Single Best: %.4f" %(sbs / n_samples)) + if (sbs - oracle) > 0: self.logger.info("Normalized Score: %.4f" %( ( par10 - oracle) / (sbs - oracle))) self.logger.debug("Selection Frequency") for algo, n in self.selection_freq.items(): - self.logger.debug("%s: %.2f" %(algo, n/(timeouts + self.solved))) + if (timeouts + self.solved) == 0: + frequency = 0 + else: + frequency = n / (timeouts + self.solved) + self.logger.debug("%s: %.2f" %(algo, frequency)) - return par10 / n_samples + return par10_out def merge(self, stat): ''' From 9b989d046f9a9a400e528c356b5ad901209ac512 Mon Sep 17 00:00:00 2001 From: Koen van der Blom Date: Thu, 28 Jan 2021 16:29:44 +0100 Subject: [PATCH 2/3] Fix issue where PCA would fail when n_components was chosen larger than the number of instances or features --- autofolio/autofolio.py | 2 +- autofolio/feature_preprocessing/pca.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/autofolio/autofolio.py b/autofolio/autofolio.py index fcf7648..6e47eb4 100644 --- a/autofolio/autofolio.py +++ b/autofolio/autofolio.py @@ -378,7 +378,7 @@ def get_cs(self, scenario: ASlibScenario, autofolio_config:dict=None): # preprocessing if autofolio_config.get("pca", True): - PCAWrapper.add_params(self.cs) + PCAWrapper.add_params(self.cs, scenario) if autofolio_config.get("impute", True): ImputerWrapper.add_params(self.cs) diff --git a/autofolio/feature_preprocessing/pca.py b/autofolio/feature_preprocessing/pca.py index 79822c8..900cd1b 100644 --- a/autofolio/feature_preprocessing/pca.py +++ b/autofolio/feature_preprocessing/pca.py @@ -20,14 +20,20 @@ class PCAWrapper(object): @staticmethod - def add_params(cs: ConfigurationSpace): + def add_params(cs: ConfigurationSpace, scenario: ASlibScenario): ''' adds parameters to ConfigurationSpace ''' pca_switch = CategoricalHyperparameter( "pca", choices=[True, False], default_value=False) + n_instances = len(scenario.instances) + n_features = len(scenario.features) + n_comp_upper_default = 20 + n_comp_upper = min(n_comp_upper_default, n_instances, n_features) + n_comp_value_default = 7 + n_comp_value = min(n_comp_value_default, n_comp_upper) n_components = UniformIntegerHyperparameter( - "pca_n_components", lower=1, upper=20, default_value=7, log=True) + "pca_n_components", lower=1, upper=n_comp_upper, default_value=n_comp_value, log=True) cs.add_hyperparameter(pca_switch) cs.add_hyperparameter(n_components) cond = InCondition( From f8adddbe22f31181b6347f2179ad2b4fef866043 Mon Sep 17 00:00:00 2001 From: Koen van der Blom Date: Thu, 28 Jan 2021 16:36:58 +0100 Subject: [PATCH 3/3] Fix par10_out being undefined for the else branch --- autofolio/validation/validate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/autofolio/validation/validate.py b/autofolio/validation/validate.py index c8fcc0a..2d07e8b 100644 --- a/autofolio/validation/validate.py +++ b/autofolio/validation/validate.py @@ -87,6 +87,7 @@ def show(self, remove_unsolvable: bool=True): self.logger.info("Number of instances: %d" %(n_samples)) self.logger.info("Average Solution Quality: %.4f" % (par1 / n_samples)) par10 = par1 + par10_out = par10 / n_samples if n_samples == 0: oracle_out = sys.maxsize