Skip to content

[BUG] TSCOptCV fails with HillClimbing optimizer #191

Closed

Description

@fkiraly

TSCOptCV fails with the HillClimbing optimizer.

To reproduce currently, check_estimator can be called:

from hyperactive.integrations.sktime import TSCOptCV
from sktime.utils import check_estimator

check_estimator(TSCOptCV, raise_exceptions=True)

There is likely a more minimal example, traceback here

File C:\Workspace\sktime\sktime\classification\base.py:264, in BaseClassifier.fit(self, X, y)
    261 X = self._convert_X(X, X_mtype)
    263 # pass coerced and checked data to inner _fit
--> [264](file:///C:/Workspace/sktime/sktime/classification/base.py:264) self._fit(X, y)
    265 self.fit_time_ = int(round(time.time() * 1000)) - start
    267 # this should happen last: fitted state is set to True

File C:\Workspace\Hyperactive\src\hyperactive\integrations\sktime\_classification.py:253, in TSCOptCV._fit(self, X, y)
    251 optimizer = self.optimizer.clone()
    252 optimizer.set_params(experiment=experiment)
--> [253](file:///C:/Workspace/Hyperactive/src/hyperactive/integrations/sktime/_classification.py:253) best_params = optimizer.solve()
    255 self.best_params_ = best_params
    256 self.best_estimator_ = estimator.set_params(**best_params)

File C:\Workspace\Hyperactive\src\hyperactive\base\_optimizer.py:82, in BaseOptimizer.solve(self)
     79 experiment = self.get_experiment()
     80 search_config = self.get_search_config()
---> [82](file:///C:/Workspace/Hyperactive/src/hyperactive/base/_optimizer.py:82) best_params = self._solve(experiment, **search_config)
     83 self.best_params_ = best_params
     84 return best_params

File C:\Workspace\Hyperactive\src\hyperactive\opt\_adapters\_gfo.py:137, in _BaseGFOadapter._solve(self, experiment, **search_config)
    134 gfopt = gfo_cls(**search_config)
    136 with StdoutMute(active=not self.verbose):
--> [137](file:///C:/Workspace/Hyperactive/src/hyperactive/opt/_adapters/_gfo.py:137)     gfopt.search(
    138         objective_function=experiment.score,
    139         n_iter=n_iter,
    140         max_time=max_time,
    141     )
    142 best_params = gfopt.best_para
    143 return best_params

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\search.py:95, in Search.search(self, objective_function, n_iter, max_time, max_score, early_stopping, memory, memory_warm_start, verbosity)
     83 self.init_search(
     84     objective_function,
     85     n_iter,
   (...)     91     verbosity,
     92 )
     94 for nth_trial in range(n_iter):
---> [95](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/search.py:95)     self.search_step(nth_trial)
     96     if self.stop.check():
     97         break

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\search.py:183, in Search.search_step(self, nth_iter)
    180 self.nth_iter = nth_iter
    182 if self.nth_iter < self.n_inits_norm:
--> [183](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/search.py:183)     self._initialization()
    185 if self.nth_iter == self.n_init_search:
    186     self.finish_initialization()

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\_times_tracker.py:27, in TimesTracker.iter_time.<locals>.wrapper(self, *args, **kwargs)
     25 def wrapper(self, *args, **kwargs):
     26     t = time.time()
---> [27](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/_times_tracker.py:27)     res = func(self, *args, **kwargs)
     28     self.iter_times.append(time.time() - t)
     29     return res

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\search.py:40, in Search._initialization(self)
     36 self.best_score = self.p_bar.score_best
     38 init_pos = self.init_pos()
---> [40](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/search.py:40) score_new = self._score(init_pos)
     41 self.evaluate_init(score_new)
     43 self.pos_l.append(init_pos)

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\_times_tracker.py:18, in TimesTracker.eval_time.<locals>.wrapper(self, *args, **kwargs)
     16 def wrapper(self, *args, **kwargs):
     17     t = time.time()
---> [18](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/_times_tracker.py:18)     res = func(self, *args, **kwargs)
     19     self.eval_times.append(time.time() - t)
     20     return res

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\search.py:32, in Search._score(self, pos)
     30 @TimesTracker.eval_time
     31 def _score(self, pos):
---> [32](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/search.py:32)     return self.score(pos)

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\_results_manager.py:33, in ResultsManager.score.<locals>._wrapper(pos)
     31 value = self.conv.position2value(pos)
     32 para = self.conv.value2para(value)
---> [33](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/_results_manager.py:33) results_dict = self._obj_func_results(objective_function, para)
     35 self.results_list.append({**results_dict, **para})
     37 return results_dict["score"]

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\_results_manager.py:16, in ResultsManager._obj_func_results(self, objective_function, para)
     15 def _obj_func_results(self, objective_function, para):
---> [16](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/_results_manager.py:16)     results = objective_function(para)
     18     if isinstance(results, tuple):
     19         score = results[0]

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\_memory.py:50, in Memory.memory.<locals>.wrapper(para)
     48 def wrapper(para: List[float]) -> float:
     49     value = self.conv.para2value(para)
---> [50](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/_memory.py:50)     position = self.conv.value2position(value)
     51     pos_tuple = tuple(position)
     53     if pos_tuple in self.memory_dict:

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\optimizers\core_optimizer\converter.py:72, in Converter.returnNoneIfArgNone.<locals>.wrapper(self, *args)
     70         return None
     71 else:
---> [72](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/optimizers/core_optimizer/converter.py:72)     return func_(self, *args)

File c:\ProgramData\anaconda3\envs\sktime-skpro-skbase-312\Lib\site-packages\gradient_free_optimizers\optimizers\core_optimizer\converter.py:89, in Converter.value2position(self, value)
     87 position = []
     88 for n, space_dim in enumerate(self.search_space_values):
---> [89](file:///C:/ProgramData/anaconda3/envs/sktime-skpro-skbase-312/Lib/site-packages/gradient_free_optimizers/optimizers/core_optimizer/converter.py:89)     pos = np.abs(value[n] - np.array(space_dim)).argmin()
     90     position.append(int(pos))
     92 return np.array(position)

UFuncTypeError: ufunc 'subtract' did not contain a loop with signature matching types (dtype('<U13'), dtype('<U13')) -> None

Activity

  1. SimonBlanke commented on Sep 19, 2025

    @SimonBlanke
    Collaborator

    I am not sure what kind of search-space or objective-function is passed to the GFO backend in "TSCOptCV", but from the error message it looks like there are categorical values in the search-space (like strings). But GFO cannot handle categorical values, yet. We abandoned native support for categorical values in the search-space in Hyperactive in v5.0.0 and decided to "outsource" it to GFO in the future.

  2. SimonBlanke commented on Sep 19, 2025

    @SimonBlanke
    Collaborator

    I get a very different error when executing the script (which is likely fixed by #190):

    FAILED: scoring value <function accuracy_score at 0x70af13831260> looks like it is a metric function rather than a scorer. A scorer should require an estimator as its first parameter. Please use `make_scorer` to convert a metric to a scorer.
    Traceback (most recent call last):
      File "/home/me/workspace/open-source/hyper_stack/Hyperactive/tsc_opt_cv_error.py", line 4, in <module>
        check_estimator(TSCOptCV, raise_exceptions=True)
      File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sktime/utils/estimator_checks.py", line 116, in check_estimator
        test_cls_results = test_cls().run_tests(
                           ^^^^^^^^^^^^^^^^^^^^^
      File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sktime/tests/test_all_estimators.py", line 639, in run_tests
        raise err
      File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sktime/tests/test_all_estimators.py", line 629, in run_tests
        test_fun(**deepcopy(args))
      File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sktime/tests/test_all_estimators.py", line 1329, in test_fit_does_not_overwrite_hyper_params
        fitted_est = scenario.run(estimator_instance, method_sequence=["fit"])
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sktime/utils/_testing/scenarios.py", line 180, in run
        res = getattr(obj, methodname)(**args)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sktime/classification/base.py", line 273, in fit
        self._fit(X, y)
      File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/hyperactive/integrations/sktime/_classification.py", line 242, in _fit
        scoring = check_scoring(DummyClassifier(), self.scoring)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sklearn/utils/_param_validation.py", line 218, in wrapper
        return func(*args, **kwargs)
               ^^^^^^^^^^^^^^^^^^^^^
      File "/home/me/anaconda3/envs/dev-py3.12/lib/python3.12/site-packages/sklearn/metrics/_scorer.py", line 1029, in check_scoring
        raise ValueError(
    ValueError: scoring value <function accuracy_score at 0x70af13831260> looks like it is a metric function rather than a scorer. A scorer should require an estimator as its first parameter. Please use `make_scorer` to convert a metric to a scorer.
    
  3. fkiraly commented on Sep 21, 2025

    @fkiraly
    ContributorAuthor

    the reported error is after the fix applied in #190 which is now merged.

    I have verified that a categorical search space was passed, which is probably the reason?

  4. fkiraly commented on Sep 22, 2025

    @fkiraly
    ContributorAuthor

    so, not a bug?

  5. SimonBlanke commented on Sep 27, 2025

    @SimonBlanke
    Collaborator

    so, not a bug?

    Does not look like it to me.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    bugSomething isn't working

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions