From e168719226106716fda662cdac6ed47f1b0f6eeb Mon Sep 17 00:00:00 2001 From: Patrick Schaefer Date: Tue, 21 May 2024 13:38:08 -0400 Subject: [PATCH 1/5] add folds option to optuna --- lightgbmlss/model.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lightgbmlss/model.py b/lightgbmlss/model.py index 33896ce..6bb4ef5 100644 --- a/lightgbmlss/model.py +++ b/lightgbmlss/model.py @@ -291,6 +291,7 @@ def hyper_opt( hp_dict: Dict, train_set: lgb.Dataset, num_boost_round=500, + folds: Optional[Union[Iterable[Tuple[np.ndarray, np.ndarray]], _LGBMBaseCrossValidator]] = None, nfold=10, early_stopping_rounds=20, max_minutes=10, @@ -311,6 +312,12 @@ def hyper_opt( Training data. num_boost_round: int Number of boosting iterations. + folds : generator or iterator of (train_idx, test_idx) tuples, scikit-learn splitter object or None, optional (default=None) + If generator or iterator, it should yield the train and test indices for each fold. + If object, it should be one of the scikit-learn splitter classes + (https://scikit-learn.org/stable/modules/classes.html#splitter-classes) + and have ``split`` method. + This argument has highest priority over other data split arguments. nfold: int Number of folds in CV. early_stopping_rounds: int @@ -387,6 +394,7 @@ def objective(trial): lgblss_param_tuning = self.cv(hyper_params, train_set, num_boost_round=num_boost_round, + folds=folds, nfold=nfold, callbacks=[pruning_callback, early_stopping_callback], seed=seed, From 3456a83feab0c4e8d499a806d8db9fdd85e81938 Mon Sep 17 00:00:00 2001 From: Patrick Schaefer Date: Tue, 21 May 2024 13:40:37 -0400 Subject: [PATCH 2/5] add n_startup_trials option to optuna --- lightgbmlss/model.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lightgbmlss/model.py b/lightgbmlss/model.py index 6bb4ef5..ee8962f 100644 --- a/lightgbmlss/model.py +++ b/lightgbmlss/model.py @@ -296,6 +296,7 @@ def hyper_opt( early_stopping_rounds=20, max_minutes=10, n_trials=None, + n_startup_trials=10, study_name=None, silence=False, seed=None, @@ -331,6 +332,8 @@ def hyper_opt( Time budget in minutes, i.e., stop study after the given number of minutes. n_trials: int The number of trials. If this argument is set to None, there is no limitation on the number of trials. + n_startup_trials: int + The random sampling is used instead of the algorithm until the given number of trials finish in the same study. study_name: str Name of the hyperparameter study. silence: bool @@ -420,7 +423,7 @@ def objective(trial): else: sampler = TPESampler() - pruner = optuna.pruners.MedianPruner(n_startup_trials=10, n_warmup_steps=20) + pruner = optuna.pruners.MedianPruner(n_startup_trials=n_startup_trials, n_warmup_steps=20) study = optuna.create_study(sampler=sampler, pruner=pruner, direction="minimize", study_name=study_name) study.optimize(objective, n_trials=n_trials, timeout=60 * max_minutes, show_progress_bar=True) From f521d8b400c79f362173b1b6ea5775e53a5bed74 Mon Sep 17 00:00:00 2001 From: Patrick Schaefer Date: Tue, 21 May 2024 14:14:22 -0400 Subject: [PATCH 3/5] formatting --- lightgbmlss/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lightgbmlss/model.py b/lightgbmlss/model.py index ee8962f..1a46662 100644 --- a/lightgbmlss/model.py +++ b/lightgbmlss/model.py @@ -291,7 +291,7 @@ def hyper_opt( hp_dict: Dict, train_set: lgb.Dataset, num_boost_round=500, - folds: Optional[Union[Iterable[Tuple[np.ndarray, np.ndarray]], _LGBMBaseCrossValidator]] = None, + folds=None, nfold=10, early_stopping_rounds=20, max_minutes=10, @@ -313,7 +313,7 @@ def hyper_opt( Training data. num_boost_round: int Number of boosting iterations. - folds : generator or iterator of (train_idx, test_idx) tuples, scikit-learn splitter object or None, optional (default=None) + folds: generator or iterator of (train_idx, test_idx) tuples, scikit-learn splitter object or None, optional (default=None) If generator or iterator, it should yield the train and test indices for each fold. If object, it should be one of the scikit-learn splitter classes (https://scikit-learn.org/stable/modules/classes.html#splitter-classes) From 3df74a20012113d614b393a75ad8ddba02d65210 Mon Sep 17 00:00:00 2001 From: Patrick Schaefer Date: Wed, 22 May 2024 10:30:05 -0400 Subject: [PATCH 4/5] added multivariate option to optuna --- lightgbmlss/model.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lightgbmlss/model.py b/lightgbmlss/model.py index 1a46662..cb0149d 100644 --- a/lightgbmlss/model.py +++ b/lightgbmlss/model.py @@ -291,12 +291,13 @@ def hyper_opt( hp_dict: Dict, train_set: lgb.Dataset, num_boost_round=500, - folds=None, + folds: Optional[Union[Iterable[Tuple[np.ndarray, np.ndarray]], _LGBMBaseCrossValidator]] = None, nfold=10, early_stopping_rounds=20, max_minutes=10, n_trials=None, n_startup_trials=10, + multivariate=False, study_name=None, silence=False, seed=None, @@ -334,6 +335,8 @@ def hyper_opt( The number of trials. If this argument is set to None, there is no limitation on the number of trials. n_startup_trials: int The random sampling is used instead of the algorithm until the given number of trials finish in the same study. + multivariate: bool + If this is True, the multivariate TPE is used when suggesting parameters. The multivariate TPE is reported to outperform the independent TPE. study_name: str Name of the hyperparameter study. silence: bool @@ -419,7 +422,7 @@ def objective(trial): optuna.logging.set_verbosity(optuna.logging.WARNING) if hp_seed is not None: - sampler = TPESampler(seed=hp_seed) + sampler = TPESampler(seed=hp_seed,multivariate=multivariate) else: sampler = TPESampler() From a4086117f320db33d7798ebd7779ed5470c82393 Mon Sep 17 00:00:00 2001 From: Patrick Schaefer Date: Tue, 11 Jun 2024 15:33:13 -0400 Subject: [PATCH 5/5] unit test --- tests/test_model/test_model.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_model/test_model.py b/tests/test_model/test_model.py index c1f12a1..a3a3815 100644 --- a/tests/test_model/test_model.py +++ b/tests/test_model/test_model.py @@ -6,6 +6,7 @@ from lightgbmlss.datasets.data_loader import load_simulated_gaussian_data import pytest from pytest import approx +from sklearn.model_selection import KFold @pytest.fixture @@ -128,6 +129,9 @@ def test_model_hpo(self, univariate_data, univariate_lgblss,): dtrain, num_boost_round=10, nfold=5, + n_startup_trials=10, + folds=KFold(n_splits=10), + multivariate=True, early_stopping_rounds=20, max_minutes=10, n_trials=5,