From f81bf2c1cfe9c7764804aeb26bec9424e58cb276 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Wed, 10 Sep 2025 19:30:36 +0200 Subject: [PATCH 01/21] v5rc1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 573f718a..381f082e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ where = ["src"] [project] name = "hyperactive" -version = "4.8.1" +version = "5.0.0rc1" description = "An optimization and data collection toolbox for convenient and fast prototyping of computationally expensive models." readme = "README.md" requires-python = ">=3.9" From a70a1aa1bfbea08899d12310c32664b385f96894 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Wed, 10 Sep 2025 19:31:17 +0200 Subject: [PATCH 02/21] v5.0.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 381f082e..8e8a7f78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ where = ["src"] [project] name = "hyperactive" -version = "5.0.0rc1" +version = "5.0.0" description = "An optimization and data collection toolbox for convenient and fast prototyping of computationally expensive models." readme = "README.md" requires-python = ">=3.9" From 35e03d3124852f8d036bf96eb8a487a4ff07bf21 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Wed, 10 Sep 2025 20:25:23 +0200 Subject: [PATCH 03/21] fix string --- src/hyperactive/utils/estimator_checks.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/hyperactive/utils/estimator_checks.py b/src/hyperactive/utils/estimator_checks.py index 1bc9f793..009869d5 100644 --- a/src/hyperactive/utils/estimator_checks.py +++ b/src/hyperactive/utils/estimator_checks.py @@ -96,14 +96,12 @@ def check_estimator( {'test_clone[HillClimbing-1]': 'PASSED'} """ msg = ( - "check_estimator is a testing utility for developers, and " - "requires pytest to be present " - "in the python environment, but pytest was not found. " - "pytest is a developer dependency and not included in the base " - "sktime installation. Please run: `pip install pytest` to " - "install the pytest package. " - "To install sktime with all developer dependencies, run:" - " `pip install hyperactive[dev]`" + "check_estimator is a testing utility for developers and requires " + "pytest to be present in the Python environment, but pytest was not found. " + "pytest is a developer dependency and not included in the base Hyperactive " + "installation. Please run: `pip install pytest` to install the pytest package. " + "To install Hyperactive with all developer dependencies, run: " + "`pip install hyperactive[dev]`." ) _check_soft_dependencies("pytest", msg=msg) From e1b2b04f8db9fd3358e28db17e021e541194f8ac Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Wed, 10 Sep 2025 20:25:50 +0200 Subject: [PATCH 04/21] handle lower and higher is better --- src/hyperactive/opt/gridsearch/_sk.py | 14 ++++++++++++-- src/hyperactive/opt/random_search.py | 12 +++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/hyperactive/opt/gridsearch/_sk.py b/src/hyperactive/opt/gridsearch/_sk.py index 8c0cd51c..de27ac81 100644 --- a/src/hyperactive/opt/gridsearch/_sk.py +++ b/src/hyperactive/opt/gridsearch/_sk.py @@ -161,11 +161,21 @@ def _solve(self, experiment, param_grid, error_score, backend, backend_params): backend_params=backend_params, ) - best_index = np.argmin(scores) + # choose selection direction based on experiment tag + hib = experiment.get_tag("property:higher_or_lower_is_better", "higher") + if hib == "lower": + best_index = int(np.argmin(scores)) + else: # default and "higher" + best_index = int(np.argmax(scores)) + best_params = candidate_params[best_index] + # store public attributes self.best_index_ = best_index - self.best_score_ = scores[best_index] + # compute signed score using experiment.score to follow the convention + signed_score, _ = experiment.score(best_params) + self.best_score_ = float(signed_score) + self.best_params_ = best_params return best_params diff --git a/src/hyperactive/opt/random_search.py b/src/hyperactive/opt/random_search.py index c6c23e10..dc2f4b60 100644 --- a/src/hyperactive/opt/random_search.py +++ b/src/hyperactive/opt/random_search.py @@ -190,12 +190,18 @@ def _solve( backend_params=backend_params, ) - best_index = int(np.argmin(scores)) # lower-is-better convention + hib = experiment.get_tag("property:higher_or_lower_is_better", "higher") + if hib == "lower": + best_index = int(np.argmin(scores)) + else: # default and "higher" + best_index = int(np.argmax(scores)) + best_params = candidate_params[best_index] - # public attributes for external consumers + # public attributes for external consumers (signed score convention) self.best_index_ = best_index - self.best_score_ = float(scores[best_index]) + signed_score, _ = experiment.score(best_params) + self.best_score_ = float(signed_score) self.best_params_ = best_params return best_params From 4514a95f4e0162203cee9dea16a46a7c9ad3e8b5 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Wed, 10 Sep 2025 20:37:12 +0200 Subject: [PATCH 05/21] handle mixed and other values --- src/hyperactive/base/_experiment.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/hyperactive/base/_experiment.py b/src/hyperactive/base/_experiment.py index f95ae560..22023bed 100644 --- a/src/hyperactive/base/_experiment.py +++ b/src/hyperactive/base/_experiment.py @@ -1,4 +1,5 @@ """Base class for experiment.""" + # copyright: hyperactive developers, MIT License (see LICENSE file) import numpy as np @@ -22,7 +23,7 @@ def __init__(self): super().__init__() def __call__(self, params): - """Score parameters. Same as score call, returns only only a first element.""" + """Score parameters. Same as score call, returns only a first element.""" score, _ = self.score(params) return score @@ -125,6 +126,15 @@ def score(self, params): sign = 1 elif hib == "lower": sign = -1 + elif hib == "mixed": + raise NotImplementedError( + "Score is undefined for mixed objectives. Override `score` or " + "set a concrete objective where higher or lower is better." + ) + else: + raise ValueError( + f"Unknown value for tag 'property:higher_or_lower_is_better': {hib}" + ) eval_res = self.evaluate(params) value = eval_res[0] From b27b32be288a43d25ae05176c7aa26d3de7f8cbe Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Wed, 10 Sep 2025 20:38:24 +0200 Subject: [PATCH 06/21] integration fixes --- .../experiment/integrations/_skl_metrics.py | 18 ++++++++++-------- .../experiment/integrations/sklearn_cv.py | 11 +++++++---- .../integrations/sktime_classification.py | 11 +++++++---- src/hyperactive/integrations/sklearn/checks.py | 11 +++++++---- src/hyperactive/opt/_common.py | 9 +++++++-- 5 files changed, 38 insertions(+), 22 deletions(-) diff --git a/src/hyperactive/experiment/integrations/_skl_metrics.py b/src/hyperactive/experiment/integrations/_skl_metrics.py index 1bd130b1..4012c6e2 100644 --- a/src/hyperactive/experiment/integrations/_skl_metrics.py +++ b/src/hyperactive/experiment/integrations/_skl_metrics.py @@ -113,16 +113,18 @@ def _guess_sign_of_sklmetric(scorer): if hasattr(scorer, "greater_is_better"): return 1 if scorer.greater_is_better else -1 - elif scorer_name in HIGHER_IS_BETTER: + if scorer_name is None: + # no name available; conservatively assume lower is better + return -1 + if scorer_name in HIGHER_IS_BETTER: return 1 if HIGHER_IS_BETTER[scorer_name] else -1 - elif scorer_name.endswith("_score"): + if scorer_name.endswith("_score"): # If the scorer name ends with "_score", we assume higher is better return 1 - elif scorer_name.endswith("_loss") or scorer_name.endswith("_deviance"): - # If the scorer name ends with "_loss", we assume lower is better - return -1 - elif scorer_name.endswith("_error"): + if scorer_name.endswith("_loss") or scorer_name.endswith("_deviance"): + # If the scorer name ends with "_loss"/"_deviance", assume lower is better return -1 - else: - # If we cannot determine the sign, we assume lower is better + if scorer_name.endswith("_error"): return -1 + # If we cannot determine the sign, assume lower is better + return -1 diff --git a/src/hyperactive/experiment/integrations/sklearn_cv.py b/src/hyperactive/experiment/integrations/sklearn_cv.py index 051edfde..b4daac0f 100644 --- a/src/hyperactive/experiment/integrations/sklearn_cv.py +++ b/src/hyperactive/experiment/integrations/sklearn_cv.py @@ -104,11 +104,14 @@ def __init__(self, estimator, X, y, scoring=None, cv=None): self.scorer_ = self._scoring # Set the sign of the scoring function - if hasattr(self._scoring, "_score"): - score_func = self._scoring._score_func + _sign_attr = getattr(self._scoring, "_sign", None) + if _sign_attr is not None: + _sign = _sign_attr + else: + score_func = getattr(self._scoring, "_score_func", self._scoring) _sign = _guess_sign_of_sklmetric(score_func) - _sign_str = "higher" if _sign == 1 else "lower" - self.set_tags(**{"property:higher_or_lower_is_better": _sign_str}) + _sign_str = "higher" if _sign == 1 else "lower" + self.set_tags(**{"property:higher_or_lower_is_better": _sign_str}) def _paramnames(self): """Return the parameter names of the search. diff --git a/src/hyperactive/experiment/integrations/sktime_classification.py b/src/hyperactive/experiment/integrations/sktime_classification.py index ab4622b8..fa5f5aac 100644 --- a/src/hyperactive/experiment/integrations/sktime_classification.py +++ b/src/hyperactive/experiment/integrations/sktime_classification.py @@ -176,11 +176,14 @@ def __init__( self._scoring = _coerce_to_scorer(scoring, "classifier") # Set the sign of the scoring function - if hasattr(self._scoring, "_score"): - score_func = self._scoring._score_func + _sign_attr = getattr(self._scoring, "_sign", None) + if _sign_attr is not None: + _sign = _sign_attr + else: + score_func = getattr(self._scoring, "_score_func", self._scoring) _sign = _guess_sign_of_sklmetric(score_func) - _sign_str = "higher" if _sign == 1 else "lower" - self.set_tags(**{"property:higher_or_lower_is_better": _sign_str}) + _sign_str = "higher" if _sign == 1 else "lower" + self.set_tags(**{"property:higher_or_lower_is_better": _sign_str}) # default handling for cv if isinstance(cv, int): diff --git a/src/hyperactive/integrations/sklearn/checks.py b/src/hyperactive/integrations/sklearn/checks.py index c752ee71..a94383af 100644 --- a/src/hyperactive/integrations/sklearn/checks.py +++ b/src/hyperactive/integrations/sklearn/checks.py @@ -1,17 +1,20 @@ """Validation checks for scikit-learn integration.""" +from functools import wraps + + class Checks: """Checks class.""" _fit_successful = False def verify_fit(function): - """Verify Fit function.""" + """Decorator to mark a successful fit while preserving signature.""" - def wrapper(self, X, y): - """Wrap function call.""" - out = function(self, X, y) + @wraps(function) + def wrapper(self, *args, **kwargs): + out = function(self, *args, **kwargs) self._fit_successful = True return out diff --git a/src/hyperactive/opt/_common.py b/src/hyperactive/opt/_common.py index 37c01e3a..b9739a42 100644 --- a/src/hyperactive/opt/_common.py +++ b/src/hyperactive/opt/_common.py @@ -4,13 +4,18 @@ def _score_params(params, meta): - """Score parameters, used in parallelization.""" + """Evaluate parameters (raw evaluate), used in parallelization. + + Returns the raw evaluate() value (not sign-adjusted), so that upstream + selection can consistently use the experiment tag to choose min/max. + """ meta = meta.copy() experiment = meta["experiment"] error_score = meta["error_score"] try: - return experiment(**params) + value, _ = experiment.evaluate(params) + return float(value) except Exception: # noqa: B904 # Catch all exceptions and assign error_score return error_score From b7484402da53dab10c993debfaf91db0670bd71c Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Thu, 11 Sep 2025 17:54:36 +0200 Subject: [PATCH 07/21] fix for when "_score_func" attribute does not exist --- .../experiment/integrations/_skl_metrics.py | 57 ++++++++++++------- .../integrations/sktime_classification.py | 12 +++- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/src/hyperactive/experiment/integrations/_skl_metrics.py b/src/hyperactive/experiment/integrations/_skl_metrics.py index 4012c6e2..ded34723 100644 --- a/src/hyperactive/experiment/integrations/_skl_metrics.py +++ b/src/hyperactive/experiment/integrations/_skl_metrics.py @@ -21,35 +21,48 @@ def _coerce_to_scorer(scoring, estimator): A sklearn scorer callable. Follows the unified sklearn scorer interface """ - from sklearn.metrics import check_scoring + from inspect import signature - # check if scoring is a scorer by checking for "estimator" in signature - if scoring is None: - if isinstance(estimator, str): - if estimator == "classifier": - from sklearn.metrics import accuracy_score + from sklearn.metrics import accuracy_score, check_scoring, make_scorer, r2_score - scoring = accuracy_score - elif estimator == "regressor": - from sklearn.metrics import r2_score + def _default_metric_for(est): + if isinstance(est, str): + if est == "classifier": + return accuracy_score + if est == "regressor": + return r2_score + # conservative fallback + return accuracy_score - scoring = r2_score + # Resolve to a sklearn scorer/callable first + if scoring is None: + # use default metric for type strings; otherwise rely on sklearn default + if isinstance(estimator, str): + scoring = _default_metric_for(estimator) + scorer = make_scorer(scoring) else: - return check_scoring(estimator) - - # check using inspect.signature for "estimator" in signature - if callable(scoring): - from inspect import signature - + scorer = check_scoring(estimator) + elif callable(scoring): + # user-provided callable if "estimator" in signature(scoring).parameters: - return scoring + scorer = scoring # passthrough scorer signature else: - from sklearn.metrics import make_scorer - - return make_scorer(scoring) + scorer = make_scorer(scoring) else: - # scoring is a string (scorer name) - return check_scoring(estimator, scoring=scoring) + # string (scorer name) + scorer = check_scoring(estimator, scoring=scoring) + + # Attach a safe metric function for downstream integrations (e.g., sktime) + metric_func = getattr(scorer, "_score_func", None) + if metric_func is None: + metric_func = _default_metric_for(estimator) + try: + setattr(scorer, "_metric_func", metric_func) + except Exception: + # if scorer is not settable (unlikely), ignore + pass + + return scorer def _guess_sign_of_sklmetric(scorer): diff --git a/src/hyperactive/experiment/integrations/sktime_classification.py b/src/hyperactive/experiment/integrations/sktime_classification.py index fa5f5aac..fd69a609 100644 --- a/src/hyperactive/experiment/integrations/sktime_classification.py +++ b/src/hyperactive/experiment/integrations/sktime_classification.py @@ -226,19 +226,25 @@ def _evaluate(self, params): estimator = self.estimator.clone().set_params(**params) + # determine metric function for sktime.evaluate via centralized coerce helper + metric_func = getattr(self._scoring, "_metric_func", None) + if metric_func is None: + # very defensive fallback (should not happen due to _coerce_to_scorer) + from sklearn.metrics import accuracy_score as metric_func # type: ignore + results = evaluate( estimator, cv=self._cv, X=self.X, y=self.y, - scoring=self._scoring._score_func, + scoring=metric_func, error_score=self.error_score, backend=self.backend, backend_params=self.backend_params, ) - metric = self._scoring._score_func - result_name = f"test_{metric.__name__}" + metric = metric_func + result_name = f"test_{getattr(metric, '__name__', 'score')}" res_float = results[result_name].mean() From fb8467422cd2040057a14b976dc37329164db1ec Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Thu, 11 Sep 2025 18:51:35 +0200 Subject: [PATCH 08/21] revert changes for sign handling --- .../experiment/integrations/sklearn_cv.py | 13 +++++-------- .../integrations/sktime_classification.py | 13 +++++-------- src/hyperactive/opt/_common.py | 9 ++++----- src/hyperactive/opt/gridsearch/_sk.py | 13 ++++--------- src/hyperactive/opt/random_search.py | 10 +++------- 5 files changed, 21 insertions(+), 37 deletions(-) diff --git a/src/hyperactive/experiment/integrations/sklearn_cv.py b/src/hyperactive/experiment/integrations/sklearn_cv.py index b4daac0f..ebfbbbc9 100644 --- a/src/hyperactive/experiment/integrations/sklearn_cv.py +++ b/src/hyperactive/experiment/integrations/sklearn_cv.py @@ -103,15 +103,12 @@ def __init__(self, estimator, X, y, scoring=None, cv=None): self._scoring = _coerce_to_scorer(scoring, self.estimator) self.scorer_ = self._scoring - # Set the sign of the scoring function - _sign_attr = getattr(self._scoring, "_sign", None) - if _sign_attr is not None: - _sign = _sign_attr - else: - score_func = getattr(self._scoring, "_score_func", self._scoring) + # Set the sign of the scoring function (rely on sklearn scorer if present) + if hasattr(self._scoring, "_score"): + score_func = self._scoring._score_func _sign = _guess_sign_of_sklmetric(score_func) - _sign_str = "higher" if _sign == 1 else "lower" - self.set_tags(**{"property:higher_or_lower_is_better": _sign_str}) + _sign_str = "higher" if _sign == 1 else "lower" + self.set_tags(**{"property:higher_or_lower_is_better": _sign_str}) def _paramnames(self): """Return the parameter names of the search. diff --git a/src/hyperactive/experiment/integrations/sktime_classification.py b/src/hyperactive/experiment/integrations/sktime_classification.py index fd69a609..352f4e71 100644 --- a/src/hyperactive/experiment/integrations/sktime_classification.py +++ b/src/hyperactive/experiment/integrations/sktime_classification.py @@ -175,15 +175,12 @@ def __init__( self._scoring = _coerce_to_scorer(scoring, "classifier") - # Set the sign of the scoring function - _sign_attr = getattr(self._scoring, "_sign", None) - if _sign_attr is not None: - _sign = _sign_attr - else: - score_func = getattr(self._scoring, "_score_func", self._scoring) + # Set the sign of the scoring function (rely on sklearn scorer if present) + if hasattr(self._scoring, "_score"): + score_func = self._scoring._score_func _sign = _guess_sign_of_sklmetric(score_func) - _sign_str = "higher" if _sign == 1 else "lower" - self.set_tags(**{"property:higher_or_lower_is_better": _sign_str}) + _sign_str = "higher" if _sign == 1 else "lower" + self.set_tags(**{"property:higher_or_lower_is_better": _sign_str}) # default handling for cv if isinstance(cv, int): diff --git a/src/hyperactive/opt/_common.py b/src/hyperactive/opt/_common.py index b9739a42..9e7c1215 100644 --- a/src/hyperactive/opt/_common.py +++ b/src/hyperactive/opt/_common.py @@ -4,18 +4,17 @@ def _score_params(params, meta): - """Evaluate parameters (raw evaluate), used in parallelization. + """Score parameters, used in parallelization. - Returns the raw evaluate() value (not sign-adjusted), so that upstream - selection can consistently use the experiment tag to choose min/max. + Uses experiment.score (via __call__), which is standardized to + "higher-is-better" across experiments. """ meta = meta.copy() experiment = meta["experiment"] error_score = meta["error_score"] try: - value, _ = experiment.evaluate(params) - return float(value) + return float(experiment(**params)) except Exception: # noqa: B904 # Catch all exceptions and assign error_score return error_score diff --git a/src/hyperactive/opt/gridsearch/_sk.py b/src/hyperactive/opt/gridsearch/_sk.py index de27ac81..835bb4ae 100644 --- a/src/hyperactive/opt/gridsearch/_sk.py +++ b/src/hyperactive/opt/gridsearch/_sk.py @@ -153,6 +153,7 @@ def _solve(self, experiment, param_grid, error_score, backend, backend_params): "error_score": error_score, } + # scores are sign-adjusted via experiment.score (higher-is-better) scores = parallelize( fun=_score_params, iter=candidate_params, @@ -161,20 +162,14 @@ def _solve(self, experiment, param_grid, error_score, backend, backend_params): backend_params=backend_params, ) - # choose selection direction based on experiment tag - hib = experiment.get_tag("property:higher_or_lower_is_better", "higher") - if hib == "lower": - best_index = int(np.argmin(scores)) - else: # default and "higher" - best_index = int(np.argmax(scores)) + # select best by maximizing standardized score + best_index = int(np.argmax(scores)) best_params = candidate_params[best_index] # store public attributes self.best_index_ = best_index - # compute signed score using experiment.score to follow the convention - signed_score, _ = experiment.score(best_params) - self.best_score_ = float(signed_score) + self.best_score_ = float(scores[best_index]) self.best_params_ = best_params return best_params diff --git a/src/hyperactive/opt/random_search.py b/src/hyperactive/opt/random_search.py index dc2f4b60..2ae97bbc 100644 --- a/src/hyperactive/opt/random_search.py +++ b/src/hyperactive/opt/random_search.py @@ -190,18 +190,14 @@ def _solve( backend_params=backend_params, ) - hib = experiment.get_tag("property:higher_or_lower_is_better", "higher") - if hib == "lower": - best_index = int(np.argmin(scores)) - else: # default and "higher" - best_index = int(np.argmax(scores)) + # select best by maximizing standardized score + best_index = int(np.argmax(scores)) best_params = candidate_params[best_index] # public attributes for external consumers (signed score convention) self.best_index_ = best_index - signed_score, _ = experiment.score(best_params) - self.best_score_ = float(signed_score) + self.best_score_ = float(scores[best_index]) self.best_params_ = best_params return best_params From ee73bcd343550fbebe84c77f04b95e3c963207eb Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Thu, 11 Sep 2025 19:37:42 +0200 Subject: [PATCH 09/21] add test --- .../integrations/sktime_classification.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/hyperactive/experiment/integrations/sktime_classification.py b/src/hyperactive/experiment/integrations/sktime_classification.py index 352f4e71..ca37577c 100644 --- a/src/hyperactive/experiment/integrations/sktime_classification.py +++ b/src/hyperactive/experiment/integrations/sktime_classification.py @@ -1,4 +1,5 @@ """Experiment adapter for sktime backtesting experiments.""" + # copyright: hyperactive developers, MIT License (see LICENSE file) import numpy as np @@ -300,7 +301,18 @@ def get_test_params(cls, parameter_set="default"): "scoring": brier_score_loss, } - return [params0, params1] + def passthrough_scorer(estimator, X, y): + return estimator.score(X, y) + + params2 = { + "estimator": DummyClassifier(strategy="prior"), + "X": X, + "y": y, + "cv": KFold(n_splits=2), + "scoring": passthrough_scorer, + } + + return [params0, params1, params2] @classmethod def _get_score_params(self): From c9d63ecdbf55b605ba45e2fb68195578dc1d6284 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Thu, 11 Sep 2025 20:05:35 +0200 Subject: [PATCH 10/21] pre-commit --- examples/hyperactive_intro.ipynb | 252 ++---------------- .../experiment/integrations/_skl_metrics.py | 5 +- .../integrations/sklearn/checks.py | 3 +- 3 files changed, 29 insertions(+), 231 deletions(-) diff --git a/examples/hyperactive_intro.ipynb b/examples/hyperactive_intro.ipynb index 5a047b32..149ce654 100644 --- a/examples/hyperactive_intro.ipynb +++ b/examples/hyperactive_intro.ipynb @@ -51,17 +51,11 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "8c428229", "metadata": {}, "outputs": [], - "source": [ - "def sphere(opt):\n", - " x = opt[\"x\"]\n", - " y = opt[\"y\"]\n", - "\n", - " return -x**2 - y**2" - ] + "source": "\"\"\"Hyperactive optimization library introduction notebook.\n\nThis notebook demonstrates unified interfaces for optimizers and experiments\nusing the Hyperactive optimization library.\n\"\"\"\n\n\ndef sphere(opt):\n \"\"\"Evaluate sphere function for optimization.\n\n Parameters\n ----------\n opt : dict\n Dictionary with 'x' and 'y' keys containing numeric values.\n\n Returns\n -------\n float\n Negative sum of squares (for maximization).\n \"\"\"\n x = opt[\"x\"]\n y = opt[\"y\"]\n\n return -(x**2) - y**2" }, { "cell_type": "markdown", @@ -139,7 +133,7 @@ "source": [ "from hyperactive.experiment.bench import Parabola\n", "\n", - "?Parabola" + "Parabola?" ] }, { @@ -319,27 +313,11 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "57110e86", "metadata": {}, "outputs": [], - "source": [ - "from hyperactive.experiment.integrations import SklearnCvExperiment\n", - "from sklearn.datasets import load_iris\n", - "from sklearn.svm import SVC\n", - "from sklearn.metrics import accuracy_score\n", - "from sklearn.model_selection import KFold\n", - "\n", - "X, y = load_iris(return_X_y=True)\n", - "\n", - "sklearn_exp = SklearnCvExperiment(\n", - " estimator=SVC(),\n", - " scoring=accuracy_score,\n", - " cv=KFold(n_splits=3, shuffle=True),\n", - " X=X,\n", - " y=y,\n", - ")" - ] + "source": "from sklearn.datasets import load_iris\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import KFold\nfrom sklearn.svm import SVC\n\nfrom hyperactive.experiment.integrations import SklearnCvExperiment\n\nX, y = load_iris(return_X_y=True)\n\nsklearn_exp = SklearnCvExperiment(\n estimator=SVC(),\n scoring=accuracy_score,\n cv=KFold(n_splits=3, shuffle=True),\n X=X,\n y=y,\n)" }, { "cell_type": "markdown", @@ -500,57 +478,19 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "ab78b796", "metadata": {}, "outputs": [], - "source": [ - "def sphere(opt):\n", - " x = opt[\"x\"]\n", - " y = opt[\"y\"]\n", - "\n", - " return -x**2 - y**2" - ] + "source": "def sphere(opt):\n \"\"\"Evaluate sphere function for optimization.\n\n Parameters\n ----------\n opt : dict\n Dictionary with 'x' and 'y' keys containing numeric values.\n\n Returns\n -------\n float\n Negative sum of squares (for maximization).\n \"\"\"\n x = opt[\"x\"]\n y = opt[\"y\"]\n\n return -(x**2) - y**2" }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "7104e5ec", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "data": { - "text/plain": [ - "{'x': np.float64(0.10101010101010033), 'y': np.float64(0.10101010101010033)}" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import numpy as np\n", - "from hyperactive.opt import HillClimbing\n", - "\n", - "hillclimbing_config = {\n", - " \"search_space\": {\n", - " \"x\": np.linspace(-10, 10, 100),\n", - " \"y\": np.linspace(-10, 10, 100),\n", - " },\n", - " \"n_iter\": 1000,\n", - "}\n", - "hill_climbing = HillClimbing(**hillclimbing_config, experiment=sphere)\n", - "\n", - "hill_climbing.solve()" - ] + "outputs": [], + "source": "import numpy as np\n\nfrom hyperactive.opt import HillClimbing\n\nhillclimbing_config = {\n \"search_space\": {\n \"x\": np.linspace(-10, 10, 100),\n \"y\": np.linspace(-10, 10, 100),\n },\n \"n_iter\": 1000,\n}\nhill_climbing = HillClimbing(**hillclimbing_config, experiment=sphere)\n\nhill_climbing.solve()" }, { "cell_type": "markdown", @@ -562,56 +502,19 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "5e2328c9", "metadata": {}, "outputs": [], - "source": [ - "from hyperactive.experiment.integrations import SklearnCvExperiment\n", - "from sklearn.datasets import load_iris\n", - "from sklearn.svm import SVC\n", - "from sklearn.metrics import accuracy_score\n", - "from sklearn.model_selection import KFold\n", - "\n", - "X, y = load_iris(return_X_y=True)\n", - "\n", - "sklearn_exp = SklearnCvExperiment(\n", - " estimator=SVC(),\n", - " scoring=accuracy_score,\n", - " cv=KFold(n_splits=3, shuffle=True),\n", - " X=X,\n", - " y=y,\n", - ")" - ] + "source": "from sklearn.datasets import load_iris\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import KFold\nfrom sklearn.svm import SVC\n\nfrom hyperactive.experiment.integrations import SklearnCvExperiment\n\nX, y = load_iris(return_X_y=True)\n\nsklearn_exp = SklearnCvExperiment(\n estimator=SVC(),\n scoring=accuracy_score,\n cv=KFold(n_splits=3, shuffle=True),\n X=X,\n y=y,\n)" }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "e9a07a73", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'C': 0.01, 'gamma': 1}" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from hyperactive.opt import GridSearchSk as GridSearch\n", - "\n", - "param_grid = {\n", - " \"C\": [0.01, 0.1, 1, 10],\n", - " \"gamma\": [0.0001, 0.01, 0.1, 1, 10],\n", - "}\n", - "grid_search = GridSearch(param_grid=param_grid, experiment=sklearn_exp)\n", - "\n", - "grid_search.solve()" - ] + "outputs": [], + "source": "from hyperactive.opt import GridSearchSk as GridSearch\n\nparam_grid = {\n \"C\": [0.01, 0.1, 1, 10],\n \"gamma\": [0.0001, 0.01, 0.1, 1, 10],\n}\ngrid_search = GridSearch(param_grid=param_grid, experiment=sklearn_exp)\n\ngrid_search.solve()" }, { "cell_type": "markdown", @@ -623,67 +526,19 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "f9a4d922", "metadata": {}, "outputs": [], - "source": [ - "from hyperactive.experiment.integrations import SklearnCvExperiment\n", - "from sklearn.datasets import load_iris\n", - "from sklearn.svm import SVC\n", - "from sklearn.metrics import accuracy_score\n", - "from sklearn.model_selection import KFold\n", - "\n", - "X, y = load_iris(return_X_y=True)\n", - "\n", - "sklearn_exp = SklearnCvExperiment(\n", - " estimator=SVC(),\n", - " scoring=accuracy_score,\n", - " cv=KFold(n_splits=3, shuffle=True),\n", - " X=X,\n", - " y=y,\n", - ")" - ] + "source": "from sklearn.datasets import load_iris\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import KFold\nfrom sklearn.svm import SVC\n\nfrom hyperactive.experiment.integrations import SklearnCvExperiment\n\nX, y = load_iris(return_X_y=True)\n\nsklearn_exp = SklearnCvExperiment(\n estimator=SVC(),\n scoring=accuracy_score,\n cv=KFold(n_splits=3, shuffle=True),\n X=X,\n y=y,\n)" }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "9a13b4f3", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "data": { - "text/plain": [ - "{'C': np.float64(10.0), 'gamma': np.float64(0.1)}" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import numpy as np\n", - "from hyperactive.opt import HillClimbing\n", - "\n", - "hillclimbing_config = {\n", - " \"search_space\": {\n", - " \"C\": np.array([0.01, 0.1, 1, 10]),\n", - " \"gamma\": np.array([0.0001, 0.01, 0.1, 1, 10]),\n", - " },\n", - " \"n_iter\": 100,\n", - "}\n", - "hill_climbing = HillClimbing(**hillclimbing_config, experiment=sklearn_exp)\n", - "\n", - "hill_climbing.solve()" - ] + "outputs": [], + "source": "import numpy as np\n\nfrom hyperactive.opt import HillClimbing\n\nhillclimbing_config = {\n \"search_space\": {\n \"C\": np.array([0.01, 0.1, 1, 10]),\n \"gamma\": np.array([0.0001, 0.01, 0.1, 1, 10]),\n },\n \"n_iter\": 100,\n}\nhill_climbing = HillClimbing(**hillclimbing_config, experiment=sklearn_exp)\n\nhill_climbing.solve()" }, { "cell_type": "markdown", @@ -716,31 +571,11 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "4bdf2d49", "metadata": {}, "outputs": [], - "source": [ - "# 1. defining the tuned estimator\n", - "from sklearn.svm import SVC\n", - "from hyperactive.integrations.sklearn import OptCV\n", - "from hyperactive.opt import GridSearchSk as GridSearch\n", - "\n", - "param_grid = {\"kernel\": [\"linear\", \"rbf\"], \"C\": [1, 10]}\n", - "tuned_svc = OptCV(SVC(), optimizer=GridSearch(param_grid))\n", - "\n", - "# 2. fitting the tuned estimator = tuning the hyperparameters\n", - "from sklearn.datasets import load_iris\n", - "from sklearn.model_selection import train_test_split\n", - "\n", - "X, y = load_iris(return_X_y=True)\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", - "\n", - "tuned_svc.fit(X_train, y_train)\n", - "\n", - "# 3. making predictions with the tuned estimator\n", - "y_pred = tuned_svc.predict(X_test)" - ] + "source": "# 1. defining the tuned estimator\nfrom sklearn.datasets import load_iris\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.svm import SVC\n\nfrom hyperactive.integrations.sklearn import OptCV\nfrom hyperactive.opt import GridSearchSk as GridSearch\n\nparam_grid = {\"kernel\": [\"linear\", \"rbf\"], \"C\": [1, 10]}\ntuned_svc = OptCV(SVC(), optimizer=GridSearch(param_grid))\n\n# 2. fitting the tuned estimator = tuning the hyperparameters\nX, y = load_iris(return_X_y=True)\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n\ntuned_svc.fit(X_train, y_train)\n\n# 3. making predictions with the tuned estimator\ny_pred = tuned_svc.predict(X_test)" }, { "cell_type": "markdown", @@ -1198,48 +1033,11 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "f606284b", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - } - ], - "source": [ - "# 1. defining the tuned estimator\n", - "from sklearn.svm import SVC\n", - "from hyperactive.integrations.sklearn import OptCV\n", - "from hyperactive.opt import HillClimbing\n", - "\n", - "# picking the optimizer is the only part that changes!\n", - "hill_climbing_config = {\n", - " \"search_space\": {\n", - " \"C\": np.array([0.01, 0.1, 1, 10]),\n", - " \"gamma\": np.array([0.0001, 0.01, 0.1, 1, 10]),\n", - " },\n", - " \"n_iter\": 100,\n", - "}\n", - "hill_climbing = HillClimbing(**hill_climbing_config)\n", - "\n", - "tuned_svc = OptCV(SVC(), optimizer=hill_climbing)\n", - "\n", - "# 2. fitting the tuned estimator = tuning the hyperparameters\n", - "from sklearn.datasets import load_iris\n", - "from sklearn.model_selection import train_test_split\n", - "\n", - "X, y = load_iris(return_X_y=True)\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", - "\n", - "tuned_svc.fit(X_train, y_train)\n", - "\n", - "# 3. making predictions with the tuned estimator\n", - "y_pred = tuned_svc.predict(X_test)" - ] + "outputs": [], + "source": "# 1. defining the tuned estimator\nfrom sklearn.datasets import load_iris\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.svm import SVC\n\nfrom hyperactive.integrations.sklearn import OptCV\nfrom hyperactive.opt import HillClimbing\n\n# picking the optimizer is the only part that changes!\nhill_climbing_config = {\n \"search_space\": {\n \"C\": np.array([0.01, 0.1, 1, 10]),\n \"gamma\": np.array([0.0001, 0.01, 0.1, 1, 10]),\n },\n \"n_iter\": 100,\n}\nhill_climbing = HillClimbing(**hill_climbing_config)\n\ntuned_svc = OptCV(SVC(), optimizer=hill_climbing)\n\n# 2. fitting the tuned estimator = tuning the hyperparameters\nX, y = load_iris(return_X_y=True)\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n\ntuned_svc.fit(X_train, y_train)\n\n# 3. making predictions with the tuned estimator\ny_pred = tuned_svc.predict(X_test)" }, { "cell_type": "markdown", diff --git a/src/hyperactive/experiment/integrations/_skl_metrics.py b/src/hyperactive/experiment/integrations/_skl_metrics.py index ded34723..5e2b3bda 100644 --- a/src/hyperactive/experiment/integrations/_skl_metrics.py +++ b/src/hyperactive/experiment/integrations/_skl_metrics.py @@ -58,8 +58,9 @@ def _default_metric_for(est): metric_func = _default_metric_for(estimator) try: setattr(scorer, "_metric_func", metric_func) - except Exception: - # if scorer is not settable (unlikely), ignore + except (TypeError, AttributeError): + # Some scorer objects may be read-only or frozen (e.g., built-in functions) + # This is acceptable as _metric_func is an optional enhancement for downstream pass return scorer diff --git a/src/hyperactive/integrations/sklearn/checks.py b/src/hyperactive/integrations/sklearn/checks.py index a94383af..93f3bf1c 100644 --- a/src/hyperactive/integrations/sklearn/checks.py +++ b/src/hyperactive/integrations/sklearn/checks.py @@ -1,6 +1,5 @@ """Validation checks for scikit-learn integration.""" - from functools import wraps @@ -10,7 +9,7 @@ class Checks: _fit_successful = False def verify_fit(function): - """Decorator to mark a successful fit while preserving signature.""" + """Mark fit successful and preserve signature.""" @wraps(function) def wrapper(self, *args, **kwargs): From 9f357a2dda8a59bdd44a4dd80aabef839116d869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20Kir=C3=A1ly?= Date: Thu, 11 Sep 2025 22:01:52 +0200 Subject: [PATCH 11/21] Update test_examples.py --- examples/test_examples.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/test_examples.py b/examples/test_examples.py index 12e147df..ad60695c 100644 --- a/examples/test_examples.py +++ b/examples/test_examples.py @@ -10,10 +10,8 @@ don't break the examples. """ -import os import sys import subprocess -import tempfile from pathlib import Path import pytest From e3bcfaec76188a88c4f08011c62b8d9cc99427e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20Kir=C3=A1ly?= Date: Thu, 11 Sep 2025 22:09:53 +0200 Subject: [PATCH 12/21] sign logic --- .../experiment/integrations/_skl_metrics.py | 81 ++++++++++++++----- .../experiment/integrations/sklearn_cv.py | 15 +--- .../integrations/sktime_classification.py | 14 +--- 3 files changed, 69 insertions(+), 41 deletions(-) diff --git a/src/hyperactive/experiment/integrations/_skl_metrics.py b/src/hyperactive/experiment/integrations/_skl_metrics.py index 5e2b3bda..34593bfb 100644 --- a/src/hyperactive/experiment/integrations/_skl_metrics.py +++ b/src/hyperactive/experiment/integrations/_skl_metrics.py @@ -3,6 +3,35 @@ __all__ = ["_coerce_to_scorer", "_guess_sign_of_sklmetric"] + +def _default_metric_for(est): + """Get a default metric function for a given estimator type. + + Parameters + ---------- + est : sklearn estimator object or str + The estimator to get a default metric for. + + Returns + ------- + metric : callable + A default metric function. + """ + from sklearn.base import is_classifier, is_regressor + from sklearn.metrics import accuracy_score, r2_score + + if isinstance(est, str): + if est == "classifier": + return accuracy_score + if est == "regressor": + return r2_score + if is_classifier(est): + return accuracy_score + if is_regressor(est): + return r2_score + return accuracy_score # safe fallback + + def _coerce_to_scorer(scoring, estimator): """Coerce scoring argument into a sklearn scorer. @@ -23,16 +52,7 @@ def _coerce_to_scorer(scoring, estimator): """ from inspect import signature - from sklearn.metrics import accuracy_score, check_scoring, make_scorer, r2_score - - def _default_metric_for(est): - if isinstance(est, str): - if est == "classifier": - return accuracy_score - if est == "regressor": - return r2_score - # conservative fallback - return accuracy_score + from sklearn.metrics import check_scoring, make_scorer # Resolve to a sklearn scorer/callable first if scoring is None: @@ -52,18 +72,39 @@ def _default_metric_for(est): # string (scorer name) scorer = check_scoring(estimator, scoring=scoring) + return scorer + + +def _coerce_to_scorer_and_sign(scoring, estimator): + """Coerce scoring argument into a sklearn scorer and determine sign. + + Parameters + ---------- + scoring : str, callable, or None + The scoring strategy to use. + estimator : estimator object or str + The estimator to use for default scoring if scoring is None. + + If str, indicates estimator type, should be one of {"classifier", "regressor"}. + + Returns + ------- + scorer : callable + A sklearn scorer callable. + Follows the unified sklearn scorer interface + sign : int + 1 if higher scores are better, -1 if lower scores are better. + """ + scorer = _coerce_to_scorer(scoring, estimator) + # Attach a safe metric function for downstream integrations (e.g., sktime) - metric_func = getattr(scorer, "_score_func", None) - if metric_func is None: - metric_func = _default_metric_for(estimator) - try: - setattr(scorer, "_metric_func", metric_func) - except (TypeError, AttributeError): - # Some scorer objects may be read-only or frozen (e.g., built-in functions) - # This is acceptable as _metric_func is an optional enhancement for downstream - pass + score_func = getattr(scorer, "_score_func", None) + if score_func is None: + score_func = _default_metric_for(estimator) - return scorer + sign = _guess_sign_of_sklmetric(score_func) + + return scorer, sign def _guess_sign_of_sklmetric(scorer): diff --git a/src/hyperactive/experiment/integrations/sklearn_cv.py b/src/hyperactive/experiment/integrations/sklearn_cv.py index ebfbbbc9..3380aa60 100644 --- a/src/hyperactive/experiment/integrations/sklearn_cv.py +++ b/src/hyperactive/experiment/integrations/sklearn_cv.py @@ -7,10 +7,7 @@ from sklearn.utils.validation import _num_samples from hyperactive.base import BaseExperiment -from hyperactive.experiment.integrations._skl_metrics import ( - _coerce_to_scorer, - _guess_sign_of_sklmetric, -) +from hyperactive.experiment.integrations._skl_metrics import _coerce_to_scorer_and_sign class SklearnCvExperiment(BaseExperiment): @@ -100,15 +97,11 @@ def __init__(self, estimator, X, y, scoring=None, cv=None): else: self._cv = cv - self._scoring = _coerce_to_scorer(scoring, self.estimator) + self._scoring , _sign = _coerce_to_scorer_and_sign(scoring, self.estimator) self.scorer_ = self._scoring - # Set the sign of the scoring function (rely on sklearn scorer if present) - if hasattr(self._scoring, "_score"): - score_func = self._scoring._score_func - _sign = _guess_sign_of_sklmetric(score_func) - _sign_str = "higher" if _sign == 1 else "lower" - self.set_tags(**{"property:higher_or_lower_is_better": _sign_str}) + _sign_str = "higher" if _sign == 1 else "lower" + self.set_tags(**{"property:higher_or_lower_is_better": _sign_str}) def _paramnames(self): """Return the parameter names of the search. diff --git a/src/hyperactive/experiment/integrations/sktime_classification.py b/src/hyperactive/experiment/integrations/sktime_classification.py index ca37577c..d2ca27dc 100644 --- a/src/hyperactive/experiment/integrations/sktime_classification.py +++ b/src/hyperactive/experiment/integrations/sktime_classification.py @@ -5,10 +5,7 @@ import numpy as np from hyperactive.base import BaseExperiment -from hyperactive.experiment.integrations._skl_metrics import ( - _coerce_to_scorer, - _guess_sign_of_sklmetric, -) +from hyperactive.experiment.integrations._skl_metrics import _coerce_to_scorer_and_sign class SktimeClassificationExperiment(BaseExperiment): @@ -174,14 +171,11 @@ def __init__( super().__init__() - self._scoring = _coerce_to_scorer(scoring, "classifier") + self._scoring, _sign = _coerce_to_scorer_and_sign(scoring, "classifier") # Set the sign of the scoring function (rely on sklearn scorer if present) - if hasattr(self._scoring, "_score"): - score_func = self._scoring._score_func - _sign = _guess_sign_of_sklmetric(score_func) - _sign_str = "higher" if _sign == 1 else "lower" - self.set_tags(**{"property:higher_or_lower_is_better": _sign_str}) + _sign_str = "higher" if _sign == 1 else "lower" + self.set_tags(**{"property:higher_or_lower_is_better": _sign_str}) # default handling for cv if isinstance(cv, int): From c0d90ebf274ca932e8d840bd74f0863fe1e09bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20Kir=C3=A1ly?= Date: Thu, 11 Sep 2025 23:01:55 +0200 Subject: [PATCH 13/21] lint --- src/hyperactive/experiment/integrations/_skl_metrics.py | 7 +++++-- src/hyperactive/experiment/integrations/sklearn_cv.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/hyperactive/experiment/integrations/_skl_metrics.py b/src/hyperactive/experiment/integrations/_skl_metrics.py index 34593bfb..eafb8dde 100644 --- a/src/hyperactive/experiment/integrations/_skl_metrics.py +++ b/src/hyperactive/experiment/integrations/_skl_metrics.py @@ -1,7 +1,10 @@ """Integration utilities for sklearn metrics with Hyperactive.""" -__all__ = ["_coerce_to_scorer", "_guess_sign_of_sklmetric"] - +__all__ = [ + "_coerce_to_scorer", + "_coerce_to_scorer_and_sign", + "_guess_sign_of_sklmetric", +] def _default_metric_for(est): diff --git a/src/hyperactive/experiment/integrations/sklearn_cv.py b/src/hyperactive/experiment/integrations/sklearn_cv.py index 3380aa60..65b6e7a1 100644 --- a/src/hyperactive/experiment/integrations/sklearn_cv.py +++ b/src/hyperactive/experiment/integrations/sklearn_cv.py @@ -97,7 +97,7 @@ def __init__(self, estimator, X, y, scoring=None, cv=None): else: self._cv = cv - self._scoring , _sign = _coerce_to_scorer_and_sign(scoring, self.estimator) + self._scoring, _sign = _coerce_to_scorer_and_sign(scoring, self.estimator) self.scorer_ = self._scoring _sign_str = "higher" if _sign == 1 else "lower" From cb71317aa726325de18ab462e078a0b6bac2ed6b Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 13 Sep 2025 16:02:08 +0200 Subject: [PATCH 14/21] add test --- src/hyperactive/tests/test_all_objects.py | 89 +++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/src/hyperactive/tests/test_all_objects.py b/src/hyperactive/tests/test_all_objects.py index ec399c64..b9bee7fc 100644 --- a/src/hyperactive/tests/test_all_objects.py +++ b/src/hyperactive/tests/test_all_objects.py @@ -348,3 +348,92 @@ def test_gfo_integration(self, object_instance): assert isinstance(best_params, dict), "Best parameters should be a dictionary" assert "C" in best_params, "Best parameters should contain 'C'" assert "gamma" in best_params, "Best parameters should contain 'gamma'" + + def test_selection_direction_backend(self, object_instance): + """Backends return argmax over standardized scores on controlled setup. + + This verifies the maximization direction using a tiny, deterministic + experiment and a deliberately poor warm start. It is scoped per-backend + to avoid brittle stochastic behavior. + """ + # Import backend bases lazily to avoid hard deps when not installed + try: + from hyperactive.opt._adapters._gfo import _BaseGFOadapter + except Exception: # pragma: no cover - import guard + + class _BaseGFOadapter: # type: ignore + pass + + try: + from hyperactive.opt._adapters._base_optuna_adapter import ( + _BaseOptunaAdapter, + ) + except Exception: # pragma: no cover - import guard + + class _BaseOptunaAdapter: # type: ignore + pass + + # set up a simple deterministic experiment with clear best vs worst + from hyperactive.experiment.bench import Ackley + + exp = Ackley(d=2) + # ackley is lower-better on evaluate; score flips sign to higher-better + poor = {"x0": 4.0, "x1": 4.0} + good = {"x0": 0.0, "x1": 0.0} + + # Helper: assert that returned params equal the known good point + def _assert_good(best_params): + assert isinstance(best_params, dict) + assert best_params == good, ( + f"Optimizer should select argmax of standardized score. " + f"Expected {good}, got {best_params}." + ) + + # Optuna adapters: use warm_start via initialize and categorical space + if isinstance(object_instance, _BaseOptunaAdapter): + inst = object_instance.clone() + cfg = { + "experiment": exp, + "param_space": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, + "n_trials": 2, + "initialize": {"warm_start": [poor, good]}, + "random_state": 0, + } + inst.set_params(**cfg) + best_params = inst.solve() + _assert_good(best_params) + return None + + # GFO adapters: pass discrete space and warm_start; keep iterations tiny + if isinstance(object_instance, _BaseGFOadapter): + inst = object_instance.clone() + cfg = { + "experiment": exp, + "search_space": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, + "n_iter": 1, + "initialize": { + "warm_start": [poor, good], + "grid": 0, + "random": 0, + "vertices": 0, + }, + # keep Bayesian-style pre-sampling tiny to avoid heavy defaults + "random_state": 0, + "verbose": False, + } + inst.set_params(**cfg) + best_params = inst.solve() + # In case the backend evaluates beyond warm starts, fall back to score check + if best_params != good: + sc_good, _ = exp.score(good) + sc_poor, _ = exp.score(poor) + sc_best, _ = exp.score(best_params) + assert sc_best >= max( + sc_good, sc_poor + ), "GFO backend did not maximize standardized score on the controlled setup." + else: + _assert_good(best_params) + return None + + # For other backends, no-op here; targeted direction tests live elsewhere + return None From 129846ccf3b6f553d5f6a71c8f87e6f940880d8e Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 13 Sep 2025 16:02:14 +0200 Subject: [PATCH 15/21] add test --- .../tests/test_selection_direction.py | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 src/hyperactive/tests/test_selection_direction.py diff --git a/src/hyperactive/tests/test_selection_direction.py b/src/hyperactive/tests/test_selection_direction.py new file mode 100644 index 00000000..163d6f6c --- /dev/null +++ b/src/hyperactive/tests/test_selection_direction.py @@ -0,0 +1,88 @@ +"""Tests to ensure selection direction is argmax on standardized scores. + +These target searchers that enumerate/sampler candidate configurations +so we can deterministically reconstruct the candidate list and verify that +the optimizer selects the configuration with the highest score (since +BaseExperiment.score is standardized to "higher-is-better"). +""" + +import numpy as np +from sklearn.model_selection import ParameterGrid, ParameterSampler + +from hyperactive.experiment.bench import Ackley +from hyperactive.opt.gridsearch._sk import GridSearchSk +from hyperactive.opt.random_search import RandomSearchSk + + +def _score_for_params(experiment, params_list): + scores = [] + for p in params_list: + sc, _ = experiment.score(p) + scores.append(float(sc)) + return np.array(scores) + + +def test_gridsearchsk_selects_argmax(): + """GridSearchSk should select argmax over candidate scores.""" + # build instances from class test params + for cfg in GridSearchSk.get_test_params(): + exp = cfg["experiment"] + # focus on deterministic benchmark (avoid CV shuffling randomness) + if not isinstance(exp, Ackley): + continue + grid = cfg["param_grid"] + + candidates = list(ParameterGrid(grid)) + scores = _score_for_params(exp, candidates) + + opt = GridSearchSk(**cfg) + best_params = opt.solve() + + expected_idx = int(np.argmax(scores)) + assert best_params == candidates[expected_idx] + # best_score_ may undergo formatting and is not necessary for direction test + + +def test_randomsearchsk_selects_argmax(): + """RandomSearchSk should select argmax on a deterministic setup (sklearn). + + Note: In certain environments, reproducing the exact sampled sequence + from `ParameterSampler` across two independent calls may be brittle. + If a mismatch is detected, this test is skipped rather than failing + spuriously. + """ + import pytest + from sklearn.model_selection import KFold + + from hyperactive.experiment.integrations import SklearnCvExperiment + + for cfg in RandomSearchSk.get_test_params(): + exp = cfg["experiment"] + # use sklearn experiment; rebuild with deterministic CV to avoid randomness + if not isinstance(exp, SklearnCvExperiment): + continue + cv = KFold(n_splits=3, shuffle=True, random_state=42) + exp_det = SklearnCvExperiment( + estimator=exp.estimator, scoring=exp.scoring, cv=cv, X=exp.X, y=exp.y + ) + + n_iter = cfg.get("n_iter", 10) + rnd = cfg.get("random_state", None) + param_distributions = cfg["param_distributions"] + + sampler = ParameterSampler( + param_distributions=param_distributions, + n_iter=n_iter, + random_state=rnd, + ) + candidates = list(sampler) + scores = _score_for_params(exp_det, candidates) + + cfg2 = cfg.copy() + cfg2["experiment"] = exp_det + opt = RandomSearchSk(**cfg2) + best_params = opt.solve() + + expected_idx = int(np.argmax(scores)) + if best_params != candidates[expected_idx]: + pytest.skip("Sampling sequence mismatch; skip argmax equivalence check.") From 549cbfce0f75e0e0b6b09bb5027e1660b93bde86 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 13 Sep 2025 16:13:42 +0200 Subject: [PATCH 16/21] fix test --- src/hyperactive/tests/test_all_objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyperactive/tests/test_all_objects.py b/src/hyperactive/tests/test_all_objects.py index b9bee7fc..e5a8cbea 100644 --- a/src/hyperactive/tests/test_all_objects.py +++ b/src/hyperactive/tests/test_all_objects.py @@ -410,7 +410,7 @@ def _assert_good(best_params): cfg = { "experiment": exp, "search_space": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, - "n_iter": 1, + "n_iter": 2, "initialize": { "warm_start": [poor, good], "grid": 0, From 047a60956340e386d1beaa101f8d4984aa31bda0 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 13 Sep 2025 16:21:22 +0200 Subject: [PATCH 17/21] remove tests --- .../tests/test_selection_direction.py | 88 ------------------- 1 file changed, 88 deletions(-) delete mode 100644 src/hyperactive/tests/test_selection_direction.py diff --git a/src/hyperactive/tests/test_selection_direction.py b/src/hyperactive/tests/test_selection_direction.py deleted file mode 100644 index 163d6f6c..00000000 --- a/src/hyperactive/tests/test_selection_direction.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Tests to ensure selection direction is argmax on standardized scores. - -These target searchers that enumerate/sampler candidate configurations -so we can deterministically reconstruct the candidate list and verify that -the optimizer selects the configuration with the highest score (since -BaseExperiment.score is standardized to "higher-is-better"). -""" - -import numpy as np -from sklearn.model_selection import ParameterGrid, ParameterSampler - -from hyperactive.experiment.bench import Ackley -from hyperactive.opt.gridsearch._sk import GridSearchSk -from hyperactive.opt.random_search import RandomSearchSk - - -def _score_for_params(experiment, params_list): - scores = [] - for p in params_list: - sc, _ = experiment.score(p) - scores.append(float(sc)) - return np.array(scores) - - -def test_gridsearchsk_selects_argmax(): - """GridSearchSk should select argmax over candidate scores.""" - # build instances from class test params - for cfg in GridSearchSk.get_test_params(): - exp = cfg["experiment"] - # focus on deterministic benchmark (avoid CV shuffling randomness) - if not isinstance(exp, Ackley): - continue - grid = cfg["param_grid"] - - candidates = list(ParameterGrid(grid)) - scores = _score_for_params(exp, candidates) - - opt = GridSearchSk(**cfg) - best_params = opt.solve() - - expected_idx = int(np.argmax(scores)) - assert best_params == candidates[expected_idx] - # best_score_ may undergo formatting and is not necessary for direction test - - -def test_randomsearchsk_selects_argmax(): - """RandomSearchSk should select argmax on a deterministic setup (sklearn). - - Note: In certain environments, reproducing the exact sampled sequence - from `ParameterSampler` across two independent calls may be brittle. - If a mismatch is detected, this test is skipped rather than failing - spuriously. - """ - import pytest - from sklearn.model_selection import KFold - - from hyperactive.experiment.integrations import SklearnCvExperiment - - for cfg in RandomSearchSk.get_test_params(): - exp = cfg["experiment"] - # use sklearn experiment; rebuild with deterministic CV to avoid randomness - if not isinstance(exp, SklearnCvExperiment): - continue - cv = KFold(n_splits=3, shuffle=True, random_state=42) - exp_det = SklearnCvExperiment( - estimator=exp.estimator, scoring=exp.scoring, cv=cv, X=exp.X, y=exp.y - ) - - n_iter = cfg.get("n_iter", 10) - rnd = cfg.get("random_state", None) - param_distributions = cfg["param_distributions"] - - sampler = ParameterSampler( - param_distributions=param_distributions, - n_iter=n_iter, - random_state=rnd, - ) - candidates = list(sampler) - scores = _score_for_params(exp_det, candidates) - - cfg2 = cfg.copy() - cfg2["experiment"] = exp_det - opt = RandomSearchSk(**cfg2) - best_params = opt.solve() - - expected_idx = int(np.argmax(scores)) - if best_params != candidates[expected_idx]: - pytest.skip("Sampling sequence mismatch; skip argmax equivalence check.") From e886dd5aa2e3471dcebdc8cf55ccf45a9af80827 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 13 Sep 2025 16:21:36 +0200 Subject: [PATCH 18/21] add sklearn to tests --- src/hyperactive/tests/test_all_objects.py | 49 +++++++++++++++-------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/src/hyperactive/tests/test_all_objects.py b/src/hyperactive/tests/test_all_objects.py index e5a8cbea..4e5de0f6 100644 --- a/src/hyperactive/tests/test_all_objects.py +++ b/src/hyperactive/tests/test_all_objects.py @@ -356,22 +356,11 @@ def test_selection_direction_backend(self, object_instance): experiment and a deliberately poor warm start. It is scoped per-backend to avoid brittle stochastic behavior. """ - # Import backend bases lazily to avoid hard deps when not installed - try: - from hyperactive.opt._adapters._gfo import _BaseGFOadapter - except Exception: # pragma: no cover - import guard - - class _BaseGFOadapter: # type: ignore - pass - - try: - from hyperactive.opt._adapters._base_optuna_adapter import ( - _BaseOptunaAdapter, - ) - except Exception: # pragma: no cover - import guard - - class _BaseOptunaAdapter: # type: ignore - pass + # Import backend bases to check optimizer type + from hyperactive.opt._adapters._gfo import _BaseGFOadapter + from hyperactive.opt._adapters._base_optuna_adapter import _BaseOptunaAdapter + from hyperactive.opt.gridsearch._sk import GridSearchSk + from hyperactive.opt.random_search import RandomSearchSk # set up a simple deterministic experiment with clear best vs worst from hyperactive.experiment.bench import Ackley @@ -435,5 +424,33 @@ def _assert_good(best_params): _assert_good(best_params) return None + # Sklearn GridSearch optimizer: test with discrete parameter grid + if isinstance(object_instance, GridSearchSk): + inst = object_instance.clone() + cfg = { + "experiment": exp, + "param_grid": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, + } + inst.set_params(**cfg) + best_params = inst.solve() + # GridSearchSk evaluates all grid combinations and selects best + _assert_good(best_params) + return None + + # Sklearn RandomSearch optimizer: test with discrete parameter distributions + if isinstance(object_instance, RandomSearchSk): + inst = object_instance.clone() + cfg = { + "experiment": exp, + "param_distributions": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, + "n_iter": 4, # Evaluate all combinations in small space + "random_state": 0, # Ensure deterministic sampling + } + inst.set_params(**cfg) + best_params = inst.solve() + # RandomSearchSk samples and selects best from evaluated points + _assert_good(best_params) + return None + # For other backends, no-op here; targeted direction tests live elsewhere return None From c88d848098a60a0517b5da129545d208d65c27c2 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 14 Sep 2025 09:23:39 +0200 Subject: [PATCH 19/21] clone().set_params chaining --- src/hyperactive/tests/test_all_objects.py | 78 +++++++++++------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/src/hyperactive/tests/test_all_objects.py b/src/hyperactive/tests/test_all_objects.py index 4e5de0f6..efe4334e 100644 --- a/src/hyperactive/tests/test_all_objects.py +++ b/src/hyperactive/tests/test_all_objects.py @@ -380,37 +380,37 @@ def _assert_good(best_params): # Optuna adapters: use warm_start via initialize and categorical space if isinstance(object_instance, _BaseOptunaAdapter): - inst = object_instance.clone() - cfg = { - "experiment": exp, - "param_space": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, - "n_trials": 2, - "initialize": {"warm_start": [poor, good]}, - "random_state": 0, - } - inst.set_params(**cfg) + inst = object_instance.clone().set_params( + **{ + "experiment": exp, + "param_space": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, + "n_trials": 2, + "initialize": {"warm_start": [poor, good]}, + "random_state": 0, + } + ) best_params = inst.solve() _assert_good(best_params) return None # GFO adapters: pass discrete space and warm_start; keep iterations tiny if isinstance(object_instance, _BaseGFOadapter): - inst = object_instance.clone() - cfg = { - "experiment": exp, - "search_space": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, - "n_iter": 2, - "initialize": { - "warm_start": [poor, good], - "grid": 0, - "random": 0, - "vertices": 0, - }, - # keep Bayesian-style pre-sampling tiny to avoid heavy defaults - "random_state": 0, - "verbose": False, - } - inst.set_params(**cfg) + inst = object_instance.clone().set_params( + **{ + "experiment": exp, + "search_space": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, + "n_iter": 2, + "initialize": { + "warm_start": [poor, good], + "grid": 0, + "random": 0, + "vertices": 0, + }, + # keep Bayesian-style pre-sampling tiny to avoid heavy defaults + "random_state": 0, + "verbose": False, + } + ) best_params = inst.solve() # In case the backend evaluates beyond warm starts, fall back to score check if best_params != good: @@ -426,12 +426,12 @@ def _assert_good(best_params): # Sklearn GridSearch optimizer: test with discrete parameter grid if isinstance(object_instance, GridSearchSk): - inst = object_instance.clone() - cfg = { - "experiment": exp, - "param_grid": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, - } - inst.set_params(**cfg) + inst = object_instance.clone().set_params( + **{ + "experiment": exp, + "param_grid": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, + } + ) best_params = inst.solve() # GridSearchSk evaluates all grid combinations and selects best _assert_good(best_params) @@ -439,14 +439,14 @@ def _assert_good(best_params): # Sklearn RandomSearch optimizer: test with discrete parameter distributions if isinstance(object_instance, RandomSearchSk): - inst = object_instance.clone() - cfg = { - "experiment": exp, - "param_distributions": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, - "n_iter": 4, # Evaluate all combinations in small space - "random_state": 0, # Ensure deterministic sampling - } - inst.set_params(**cfg) + inst = object_instance.clone().set_params( + **{ + "experiment": exp, + "param_distributions": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, + "n_iter": 4, # Evaluate all combinations in small space + "random_state": 0, # Ensure deterministic sampling + } + ) best_params = inst.solve() # RandomSearchSk samples and selects best from evaluated points _assert_good(best_params) From 0b302cd555226e5a96edb149aa63400f9a012210 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 14 Sep 2025 09:30:39 +0200 Subject: [PATCH 20/21] common config keys to reduce hard-coding --- src/hyperactive/tests/test_all_objects.py | 35 +++++++++++++++-------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/hyperactive/tests/test_all_objects.py b/src/hyperactive/tests/test_all_objects.py index efe4334e..3ac060ac 100644 --- a/src/hyperactive/tests/test_all_objects.py +++ b/src/hyperactive/tests/test_all_objects.py @@ -362,6 +362,21 @@ def test_selection_direction_backend(self, object_instance): from hyperactive.opt.gridsearch._sk import GridSearchSk from hyperactive.opt.random_search import RandomSearchSk + # small helper to attach the right space key without per-estimator branching + def _cfg_with_space(est, exp, space): + cfg = {"experiment": exp} + param_keys = set(est.get_params().keys()) + for key in ( + "param_space", + "search_space", + "param_grid", + "param_distributions", + ): + if key in param_keys: + cfg[key] = space + break + return cfg + # set up a simple deterministic experiment with clear best vs worst from hyperactive.experiment.bench import Ackley @@ -378,12 +393,15 @@ def _assert_good(best_params): f"Expected {good}, got {best_params}." ) + space = {"x0": [0.0, 4.0], "x1": [0.0, 4.0]} + base_cfg = _cfg_with_space(object_instance, exp, space) + # Optuna adapters: use warm_start via initialize and categorical space if isinstance(object_instance, _BaseOptunaAdapter): + inst = object_instance.clone().set_params( **{ - "experiment": exp, - "param_space": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, + **base_cfg, "n_trials": 2, "initialize": {"warm_start": [poor, good]}, "random_state": 0, @@ -397,8 +415,7 @@ def _assert_good(best_params): if isinstance(object_instance, _BaseGFOadapter): inst = object_instance.clone().set_params( **{ - "experiment": exp, - "search_space": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, + **base_cfg, "n_iter": 2, "initialize": { "warm_start": [poor, good], @@ -426,12 +443,7 @@ def _assert_good(best_params): # Sklearn GridSearch optimizer: test with discrete parameter grid if isinstance(object_instance, GridSearchSk): - inst = object_instance.clone().set_params( - **{ - "experiment": exp, - "param_grid": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, - } - ) + inst = object_instance.clone().set_params(**base_cfg) best_params = inst.solve() # GridSearchSk evaluates all grid combinations and selects best _assert_good(best_params) @@ -441,8 +453,7 @@ def _assert_good(best_params): if isinstance(object_instance, RandomSearchSk): inst = object_instance.clone().set_params( **{ - "experiment": exp, - "param_distributions": {"x0": [0.0, 4.0], "x1": [0.0, 4.0]}, + **base_cfg, "n_iter": 4, # Evaluate all combinations in small space "random_state": 0, # Ensure deterministic sampling } From 8cad42c2bf4c45fe6d4a66d7a55ccb8133d07008 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 14 Sep 2025 14:35:30 +0200 Subject: [PATCH 21/21] pre-commit --- src/hyperactive/tests/test_all_objects.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/hyperactive/tests/test_all_objects.py b/src/hyperactive/tests/test_all_objects.py index 3ac060ac..559af35d 100644 --- a/src/hyperactive/tests/test_all_objects.py +++ b/src/hyperactive/tests/test_all_objects.py @@ -357,8 +357,8 @@ def test_selection_direction_backend(self, object_instance): to avoid brittle stochastic behavior. """ # Import backend bases to check optimizer type - from hyperactive.opt._adapters._gfo import _BaseGFOadapter from hyperactive.opt._adapters._base_optuna_adapter import _BaseOptunaAdapter + from hyperactive.opt._adapters._gfo import _BaseGFOadapter from hyperactive.opt.gridsearch._sk import GridSearchSk from hyperactive.opt.random_search import RandomSearchSk @@ -398,7 +398,6 @@ def _assert_good(best_params): # Optuna adapters: use warm_start via initialize and categorical space if isinstance(object_instance, _BaseOptunaAdapter): - inst = object_instance.clone().set_params( **{ **base_cfg, @@ -434,9 +433,7 @@ def _assert_good(best_params): sc_good, _ = exp.score(good) sc_poor, _ = exp.score(poor) sc_best, _ = exp.score(best_params) - assert sc_best >= max( - sc_good, sc_poor - ), "GFO backend did not maximize standardized score on the controlled setup." + assert sc_best >= max(sc_good, sc_poor) else: _assert_good(best_params) return None